{ "config_general": { "lighteval_sha": "?", "num_fewshot_seeds": 1, "max_samples": null, "job_id": "0", "start_time": 796668.90423246, "end_time": 807588.555648873, "total_evaluation_time_secondes": "10919.65141641302", "model_config": { "model_name": "/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010", "generation_parameters": { "num_blocks": null, "block_size": null, "early_stopping": null, "repetition_penalty": null, "frequency_penalty": null, "length_penalty": null, "presence_penalty": null, "max_new_tokens": null, "min_new_tokens": null, "seed": null, "stop_tokens": null, "temperature": 0, "top_k": null, "min_p": null, "top_p": null, "truncate_prompt": null, "cache_implementation": null, "response_format": null, "skip_special_tokens": true }, "system_prompt": null, "cache_dir": "~/.cache/huggingface/lighteval", "tokenizer": null, "revision": "main", "dtype": "bfloat16", "tensor_parallel_size": 1, "data_parallel_size": 1, "pipeline_parallel_size": 1, "gpu_memory_utilization": 0.85, "enable_prefix_caching": null, "max_model_length": 32768, "quantization": null, "load_format": null, "swap_space": 4, "seed": 1234, "trust_remote_code": false, "add_special_tokens": true, "multichoice_continuations_start_space": true, "pairwise_tokenization": false, "max_num_seqs": 128, "max_num_batched_tokens": 2048, "subfolder": null, "is_async": false, "override_chat_template": null, "enforce_eager": true }, "model_name": "/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010" }, "results": { "mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0": { "em_relaxed": 0.17843388960205392, "em_relaxed_stderr": 0.013726826073077934, "em": 0.055198973042362, "em_stderr": 0.00818739755268085, "em_conservative": 0.17715019255455713, "em_conservative_stderr": 0.013688041138952063 }, "mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0": { "em_relaxed": 0.06166219839142091, "em_relaxed_stderr": 0.012471469772330128, "em": 0.005361930294906166, "em_stderr": 0.003786357797685454, "em_conservative": 0.058981233243967826, "em_conservative_stderr": 0.012214750400069162 }, "financial_exams.prompt2_completion_sk|0": { "em_relaxed": 0.2383808095952024, "em_relaxed_stderr": 0.011670501255680532, "em": 0.07571214392803598, "em_stderr": 0.007245549384888845, "em_conservative": 0.20539730134932535, "em_conservative_stderr": 0.011065150105060274 }, "mmlu.prompt1_mqa_single_correct_sk:machine_learning|0": { "em_relaxed": 0.11607142857142858, "em_relaxed_stderr": 0.03040253249894973, "em": 0.008928571428571428, "em_stderr": 0.008928571428571423, "em_conservative": 0.10714285714285714, "em_conservative_stderr": 0.02935696201881377 }, "belebele.prompt3_mqa_single_correct_sk|0": { "em_relaxed": 0.21333333333333335, "em_relaxed_stderr": 0.013662962863011163, "em": 0.03, "em_stderr": 0.005689402360648914, "em_conservative": 0.2111111111111111, "em_conservative_stderr": 0.013610798969328486 }, "mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0": { "em_relaxed": 0.15444015444015444, "em_relaxed_stderr": 0.022497923516297567, "em": 0.011583011583011582, "em_stderr": 0.006661484058899665, "em_conservative": 0.15444015444015444, "em_conservative_stderr": 0.022497923516297567 }, "mmlu.prompt3_mqa_single_correct_sk:professional_law|0": { "em_relaxed": 0.1340734415029889, "em_relaxed_stderr": 0.009961367847349073, "em": 0.0017079419299743809, "em_stderr": 0.0012071811002646103, "em_conservative": 0.12809564474807855, "em_conservative_stderr": 0.009770317447740377 }, "mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0": { "em_relaxed": 0.15, "em_relaxed_stderr": 0.035887028128263714, "em": 0.01, "em_stderr": 0.009999999999999998, "em_conservative": 0.11, "em_conservative_stderr": 0.031446603773522035 }, "mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0": { "em_relaxed": 0.15217391304347827, "em_relaxed_stderr": 0.02165993479655374, "em": 0.0036231884057971015, "em_stderr": 0.0036231884057970937, "em_conservative": 0.14130434782608695, "em_conservative_stderr": 0.02100540442245809 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0": { "em_relaxed": 0.12280701754385964, "em_relaxed_stderr": 0.04385964912280699, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.12280701754385964, "em_conservative_stderr": 0.04385964912280699 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0": { "em_relaxed": 0.08585858585858586, "em_relaxed_stderr": 0.019960225563172885, "em": 0.010101010101010102, "em_stderr": 0.007124341521250813, "em_conservative": 0.08080808080808081, "em_conservative_stderr": 0.019417681889724533 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0": { "em_relaxed": 0.21052631578947367, "em_relaxed_stderr": 0.05447885994808057, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.21052631578947367, "em_conservative_stderr": 0.05447885994808057 }, "mmlu.prompt2_mqa_single_correct_sk:professional_law|0": { "em_relaxed": 0.14175918018787362, "em_relaxed_stderr": 0.01019734819812428, "em": 0.004269854824935952, "em_stderr": 0.001906270167826036, "em_conservative": 0.14175918018787362, "em_conservative_stderr": 0.01019734819812428 }, "sklep.prompt1_qa_sk:question-answering|0": { "exact_match_slk_prefix": 0.16456224564332672, "exact_match_slk_prefix_stderr": 0.003787861300566259, "f1_slk": 0.3730151221096053, "f1_slk_stderr": 0.003855179221962441 }, "mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0": { "em_relaxed": 0.14, "em_relaxed_stderr": 0.03487350880197772, "em": 0.09, "em_stderr": 0.028762349126466143, "em_conservative": 0.14, "em_conservative_stderr": 0.03487350880197772 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0": { "em_relaxed": 0.17511520737327188, "em_relaxed_stderr": 0.025860189581375218, "em": 0.02304147465437788, "em_stderr": 0.010208603193741356, "em_conservative": 0.17511520737327188, "em_conservative_stderr": 0.025860189581375218 }, "mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0": { "em_relaxed": 0.04040404040404041, "em_relaxed_stderr": 0.019890402239668436, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.030303030303030304, "em_conservative_stderr": 0.017316017316017295 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0": { "em_relaxed": 0.14666666666666667, "em_relaxed_stderr": 0.02898223889753507, "em": 0.04, "em_stderr": 0.016053601490175616, "em_conservative": 0.14666666666666667, "em_conservative_stderr": 0.02898223889753507 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0": { "em_relaxed": 0.1822429906542056, "em_relaxed_stderr": 0.026451344562470138, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.17289719626168223, "em_conservative_stderr": 0.025910984835548666 }, "mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0": { "em_relaxed": 0.19696969696969696, "em_relaxed_stderr": 0.024523802005628457, "em": 0.003787878787878788, "em_stderr": 0.003787878787878786, "em_conservative": 0.19696969696969696, "em_conservative_stderr": 0.024523802005628457 }, "slovakcopa.prompt3_copa_sk|0": { "em_relaxed": 0.086, "em_relaxed_stderr": 0.01255081854271602, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.084, "em_conservative_stderr": 0.01241758401564368 }, "movie_review.prompt2_sentiment_binary_sk|0": { "em_norm_relaxed": 0.48306772908366535, "em_norm_relaxed_stderr": 0.011154424009707972, "em_norm": 0.01045816733067729, "em_norm_stderr": 0.002270759742696006, "em_conservative_norm": 0.47758964143426297, "em_conservative_norm_stderr": 0.011149609394535145 }, "sklep.prompt2_qa_sk:question-answering|0": { "exact_match_slk_prefix": 0.16518835437754356, "exact_match_slk_prefix_stderr": 0.003793637935655721, "f1_slk": 0.3751701402109962, "f1_slk_stderr": 0.0038611217329234445 }, "mmlu.prompt1_mqa_single_correct_sk:college_biology|0": { "em_relaxed": 0.15492957746478872, "em_relaxed_stderr": 0.030472217725828407, "em": 0.028169014084507043, "em_stderr": 0.013933866272175799, "em_conservative": 0.15492957746478872, "em_conservative_stderr": 0.030472217725828407 }, "mmlu.prompt3_mqa_single_correct_sk:nutrition|0": { "em_relaxed": 0.18811881188118812, "em_relaxed_stderr": 0.022488416775417838, "em": 0.036303630363036306, "em_stderr": 0.010763206393659715, "em_conservative": 0.18811881188118812, "em_conservative_stderr": 0.022488416775417838 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0": { "em_relaxed": 0.24822695035460993, "em_relaxed_stderr": 0.025770015644290406, "em": 0.04964539007092199, "em_stderr": 0.012957736571882147, "em_conservative": 0.24113475177304963, "em_conservative_stderr": 0.02551873104953778 }, "arc.prompt3_mqa_single_correct_sk|0": { "em_relaxed": 0.1770744225834046, "em_relaxed_stderr": 0.011169578281264629, "em": 0.022241231822070145, "em_stderr": 0.004314931013145922, "em_conservative": 0.17621899059024806, "em_conservative_stderr": 0.011148355810710043 }, "mmlu.prompt3_mqa_single_correct_sk:econometrics|0": { "em_relaxed": 0.10714285714285714, "em_relaxed_stderr": 0.02935696201881377, "em": 0.017857142857142856, "em_stderr": 0.012569900164490043, "em_conservative": 0.09821428571428571, "em_conservative_stderr": 0.02824733740077989 }, "mmlu.prompt2_mqa_single_correct_sk:prehistory|0": { "em_relaxed": 0.18808777429467086, "em_relaxed_stderr": 0.021913980163123535, "em": 0.06583072100313479, "em_stderr": 0.013906362189890747, "em_conservative": 0.18808777429467086, "em_conservative_stderr": 0.021913980163123535 }, "toxic_sk.prompt3_toxicity_sk|0": { "em_norm_relaxed": 0.4960272417707151, "em_norm_relaxed_stderr": 0.016854464514836336, "em_norm": 0.007945516458569807, "em_norm_stderr": 0.00299286751118699, "em_conservative_norm": 0.4960272417707151, "em_conservative_norm_stderr": 0.016854464514836336 }, "belebele.prompt1_mqa_single_correct_sk|0": { "em_relaxed": 0.21555555555555556, "em_relaxed_stderr": 0.01371452783236968, "em": 0.02, "em_stderr": 0.004669261421755523, "em_conservative": 0.21555555555555556, "em_conservative_stderr": 0.01371452783236968 }, "arc.prompt2_mqa_single_correct_sk|0": { "em_relaxed": 0.22412318220701455, "em_relaxed_stderr": 0.012201644195165715, "em": 0.03507271171941831, "em_stderr": 0.005382828530594407, "em_conservative": 0.223267750213858, "em_conservative_stderr": 0.012185048029719049 }, "multi_wiki_qa.prompt2_qa_sk|0": { "exact_match_slk_prefix": 0.36001600960576347, "exact_match_slk_prefix_stderr": 0.006791008078277957, "f1_slk": 0.5090494926903776, "f1_slk_stderr": 0.005852156127470673 }, "mmlu.prompt2_mqa_single_correct_sk:management|0": { "em_relaxed": 0.15, "em_relaxed_stderr": 0.035887028128263686, "em": 0.04, "em_stderr": 0.019694638556693237, "em_conservative": 0.15, "em_conservative_stderr": 0.035887028128263686 }, "mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0": { "em_relaxed": 0.06, "em_relaxed_stderr": 0.023868325657594173, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.06, "em_conservative_stderr": 0.023868325657594173 }, "mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0": { "em_relaxed": 0.08, "em_relaxed_stderr": 0.027265992434429093, "em": 0.04, "em_stderr": 0.019694638556693227, "em_conservative": 0.08, "em_conservative_stderr": 0.027265992434429093 }, "sklep.prompt3_nli_3_classes_sk:nli|0": { "em_norm_relaxed": 0.001398880895283773, "em_norm_relaxed_stderr": 0.0005284101391342323, "em_norm": 0.0, "em_norm_stderr": 0.0, "em_conservative_norm": 0.001398880895283773, "em_conservative_norm_stderr": 0.0005284101391342323 }, "mmlu.prompt1_mqa_single_correct_sk:college_medicine|0": { "em_relaxed": 0.1511627906976744, "em_relaxed_stderr": 0.027392810919920886, "em": 0.011627906976744186, "em_stderr": 0.008198095184581854, "em_conservative": 0.1511627906976744, "em_conservative_stderr": 0.027392810919920886 }, "mmlu.prompt2_mqa_single_correct_sk:business_ethics|0": { "em_relaxed": 0.22, "em_relaxed_stderr": 0.0416333199893227, "em": 0.03, "em_stderr": 0.017144660799776522, "em_conservative": 0.21, "em_conservative_stderr": 0.04093601807403326 }, "mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0": { "em_relaxed": 0.15, "em_relaxed_stderr": 0.035887028128263714, "em": 0.03, "em_stderr": 0.01714466079977653, "em_conservative": 0.11, "em_conservative_stderr": 0.031446603773522035 }, "mmlu.prompt3_mqa_single_correct_sk:international_law|0": { "em_relaxed": 0.15254237288135594, "em_relaxed_stderr": 0.03324002290477697, "em": 0.0423728813559322, "em_stderr": 0.018622984668462274, "em_conservative": 0.15254237288135594, "em_conservative_stderr": 0.03324002290477697 }, "mmlu.prompt1_mqa_single_correct_sk:international_law|0": { "em_relaxed": 0.13559322033898305, "em_relaxed_stderr": 0.03165083260501198, "em": 0.03389830508474576, "em_stderr": 0.016730444637044904, "em_conservative": 0.13559322033898305, "em_conservative_stderr": 0.03165083260501198 }, "truthfulqa.prompt3_mqa_single_correct_sk:mc1|0": { "em_relaxed": 0.011568123393316195, "em_relaxed_stderr": 0.00383613885044431, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.011568123393316195, "em_conservative_stderr": 0.00383613885044431 }, "mmlu.prompt3_mqa_single_correct_sk:prehistory|0": { "em_relaxed": 0.15047021943573669, "em_relaxed_stderr": 0.020049383410796876, "em": 0.04075235109717868, "em_stderr": 0.011087359931149772, "em_conservative": 0.14733542319749215, "em_conservative_stderr": 0.019876006858615635 }, "mmlu.prompt1_mqa_single_correct_sk:nutrition|0": { "em_relaxed": 0.1848184818481848, "em_relaxed_stderr": 0.022335536574954167, "em": 0.036303630363036306, "em_stderr": 0.010763206393659719, "em_conservative": 0.1848184818481848, "em_conservative_stderr": 0.022335536574954167 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0": { "em_relaxed": 0.09090909090909091, "em_relaxed_stderr": 0.020482086775424218, "em": 0.005050505050505051, "em_stderr": 0.00505050505050505, "em_conservative": 0.08585858585858586, "em_conservative_stderr": 0.01996022556317289 }, "mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0": { "em_relaxed": 0.09361702127659574, "em_relaxed_stderr": 0.01904256081095342, "em": 0.03829787234042553, "em_stderr": 0.012545838230269625, "em_conservative": 0.09361702127659574, "em_conservative_stderr": 0.01904256081095342 }, "mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0": { "em_relaxed": 0.17, "em_relaxed_stderr": 0.03775251680686371, "em": 0.01, "em_stderr": 0.009999999999999995, "em_conservative": 0.12, "em_conservative_stderr": 0.032659863237109066 }, "mmlu.prompt3_mqa_single_correct_sk:college_medicine|0": { "em_relaxed": 0.1569767441860465, "em_relaxed_stderr": 0.027818863343493188, "em": 0.040697674418604654, "em_stderr": 0.01511000089316197, "em_conservative": 0.1569767441860465, "em_conservative_stderr": 0.027818863343493188 }, "mmlu.prompt2_mqa_single_correct_sk:college_medicine|0": { "em_relaxed": 0.20930232558139536, "em_relaxed_stderr": 0.031109583909764628, "em": 0.06395348837209303, "em_stderr": 0.018710387348210598, "em_conservative": 0.20930232558139536, "em_conservative_stderr": 0.031109583909764628 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0": { "em_relaxed": 0.18470149253731344, "em_relaxed_stderr": 0.016777090030491517, "em": 0.03731343283582089, "em_stderr": 0.008194041665419557, "em_conservative": 0.18470149253731344, "em_conservative_stderr": 0.016777090030491517 }, "sklep.prompt2_nli_binary_sk:rte|0": { "em_norm_relaxed": 0.3301204819277108, "em_norm_relaxed_stderr": 0.011545464009906971, "em_norm": 0.0018072289156626507, "em_norm_stderr": 0.001042774975713993, "em_conservative_norm": 0.3301204819277108, "em_conservative_norm_stderr": 0.011545464009906971 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0": { "em_relaxed": 0.14285714285714285, "em_relaxed_stderr": 0.0238095238095238, "em": 0.004608294930875576, "em_stderr": 0.004608294930875571, "em_conservative": 0.14285714285714285, "em_conservative_stderr": 0.0238095238095238 }, "sklep.prompt2_hate_speech_sk:hate-speech|0": { "em_norm_relaxed": 0.7247915087187263, "em_norm_relaxed_stderr": 0.012302114305862654, "em_norm": 0.0, "em_norm_stderr": 0.0, "em_conservative_norm": 0.7247915087187263, "em_conservative_norm_stderr": 0.012302114305862654 }, "mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0": { "em_relaxed": 0.13761467889908258, "em_relaxed_stderr": 0.03314905716923366, "em": 0.045871559633027525, "em_stderr": 0.020130904850286974, "em_conservative": 0.13761467889908258, "em_conservative_stderr": 0.03314905716923366 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0": { "em_relaxed": 0.22388059701492538, "em_relaxed_stderr": 0.01802169384796668, "em": 0.04477611940298507, "em_stderr": 0.008941264053122127, "em_conservative": 0.22388059701492538, "em_conservative_stderr": 0.01802169384796668 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0": { "em_relaxed": 0.19298245614035087, "em_relaxed_stderr": 0.05273590328947574, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.19298245614035087, "em_conservative_stderr": 0.05273590328947574 }, "mmlu.prompt3_mqa_single_correct_sk:marketing|0": { "em_relaxed": 0.1623931623931624, "em_relaxed_stderr": 0.02416161812798774, "em": 0.05555555555555555, "em_stderr": 0.01500631280644691, "em_conservative": 0.1623931623931624, "em_conservative_stderr": 0.02416161812798774 }, "mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0": { "em_relaxed": 0.17721518987341772, "em_relaxed_stderr": 0.030475004517281346, "em": 0.0379746835443038, "em_stderr": 0.015254237576434418, "em_conservative": 0.17721518987341772, "em_conservative_stderr": 0.030475004517281346 }, "mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0": { "em_relaxed": 0.14042553191489363, "em_relaxed_stderr": 0.022712077616627854, "em": 0.0851063829787234, "em_stderr": 0.018241411344572286, "em_conservative": 0.14042553191489363, "em_conservative_stderr": 0.022712077616627854 }, "mmlu.prompt2_mqa_single_correct_sk:virology|0": { "em_relaxed": 0.16463414634146342, "em_relaxed_stderr": 0.029047216079384597, "em": 0.03048780487804878, "em_stderr": 0.013466226472535378, "em_conservative": 0.16463414634146342, "em_conservative_stderr": 0.029047216079384597 }, "mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0": { "em_relaxed": 0.14, "em_relaxed_stderr": 0.034873508801977704, "em": 0.01, "em_stderr": 0.009999999999999997, "em_conservative": 0.14, "em_conservative_stderr": 0.034873508801977704 }, "mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0": { "em_relaxed": 0.006675567423230975, "em_relaxed_stderr": 0.0029774114441068625, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.0053404539385847796, "em_conservative_stderr": 0.0026648668562206292 }, "mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0": { "em_relaxed": 0.25252525252525254, "em_relaxed_stderr": 0.043887175543277195, "em": 0.030303030303030304, "em_stderr": 0.01731601731601731, "em_conservative": 0.24242424242424243, "em_conservative_stderr": 0.04329004329004326 }, "mmlu.prompt1_mqa_single_correct_sk:world_religions|0": { "em_relaxed": 0.05263157894736842, "em_relaxed_stderr": 0.017126088775157082, "em": 0.023391812865497075, "em_stderr": 0.01159224790573493, "em_conservative": 0.05263157894736842, "em_conservative_stderr": 0.017126088775157082 }, "senti_sk.prompt3_sentiment_binary_sk|0": { "em_norm_relaxed": 0.25160944206008584, "em_norm_relaxed_stderr": 0.0071080082101389235, "em_norm": 0.03165236051502146, "em_norm_stderr": 0.0028677349189843607, "em_conservative_norm": 0.1990343347639485, "em_conservative_norm_stderr": 0.0065402033880141534 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0": { "em_relaxed": 0.13333333333333333, "em_relaxed_stderr": 0.027848530325897826, "em": 0.02666666666666667, "em_stderr": 0.013198422611557032, "em_conservative": 0.13333333333333333, "em_conservative_stderr": 0.027848530325897826 }, "sklep.prompt1_nli_3_classes_sk:nli|0": { "em_norm_relaxed": 0.016786570743405275, "em_norm_relaxed_stderr": 0.0018163086111764766, "em_norm": 0.0003996802557953637, "em_norm_stderr": 0.00028258837305266234, "em_conservative_norm": 0.016386890487609912, "em_conservative_norm_stderr": 0.0017949203375471519 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0": { "em_relaxed": 0.12919896640826872, "em_relaxed_stderr": 0.017072429785340966, "em": 0.012919896640826873, "em_stderr": 0.005747937884708704, "em_conservative": 0.12661498708010335, "em_conservative_stderr": 0.016925900082204414 }, "mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0": { "em_relaxed": 0.01335113484646195, "em_relaxed_stderr": 0.004196522950335422, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.0053404539385847796, "em_conservative_stderr": 0.0026648668562206297 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0": { "em_relaxed": 0.11616161616161616, "em_relaxed_stderr": 0.022828881775249377, "em": 0.020202020202020204, "em_stderr": 0.01002380379631872, "em_conservative": 0.1111111111111111, "em_conservative_stderr": 0.022390787638216763 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0": { "em_relaxed": 0.32124352331606215, "em_relaxed_stderr": 0.033699508685490674, "em": 0.046632124352331605, "em_stderr": 0.015216761819262577, "em_conservative": 0.32124352331606215, "em_conservative_stderr": 0.033699508685490674 }, "mmlu.prompt3_mqa_single_correct_sk:business_ethics|0": { "em_relaxed": 0.13, "em_relaxed_stderr": 0.03379976689896308, "em": 0.01, "em_stderr": 0.009999999999999998, "em_conservative": 0.13, "em_conservative_stderr": 0.03379976689896308 }, "mmlu.prompt2_mqa_single_correct_sk:public_relations|0": { "em_relaxed": 0.23853211009174313, "em_relaxed_stderr": 0.041009771029340336, "em": 0.09174311926605505, "em_stderr": 0.02777660875318803, "em_conservative": 0.23853211009174313, "em_conservative_stderr": 0.041009771029340336 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0": { "em_relaxed": 0.16355140186915887, "em_relaxed_stderr": 0.025342938086817392, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.1542056074766355, "em_conservative_stderr": 0.024745297661074854 }, "senti_sk.prompt2_sentiment_binary_sk|0": { "em_norm_relaxed": 0.4965128755364807, "em_norm_relaxed_stderr": 0.008189921623320459, "em_norm": 0.0037553648068669528, "em_norm_stderr": 0.001001911513818721, "em_conservative_norm": 0.49302575107296137, "em_conservative_norm_stderr": 0.008189324033951761 }, "mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0": { "em_relaxed": 0.18, "em_relaxed_stderr": 0.03861229196653694, "em": 0.07, "em_stderr": 0.02564323999762429, "em_conservative": 0.18, "em_conservative_stderr": 0.03861229196653694 }, "belebele.prompt2_mqa_single_correct_sk|0": { "em_relaxed": 0.2111111111111111, "em_relaxed_stderr": 0.013610798969328482, "em": 0.016666666666666666, "em_stderr": 0.004269675894690937, "em_conservative": 0.2111111111111111, "em_conservative_stderr": 0.013610798969328482 }, "mmlu.prompt1_mqa_single_correct_sk:virology|0": { "em_relaxed": 0.12195121951219512, "em_relaxed_stderr": 0.025630589418441155, "em": 0.03048780487804878, "em_stderr": 0.013466226472535378, "em_conservative": 0.12195121951219512, "em_conservative_stderr": 0.025630589418441155 }, "sklep.prompt2_nli_3_classes_sk:nli|0": { "em_norm_relaxed": 0.09972022382094324, "em_norm_relaxed_stderr": 0.004236089409386789, "em_norm": 0.0, "em_norm_stderr": 0.0, "em_conservative_norm": 0.09952038369304557, "em_conservative_norm_stderr": 0.004232312358957502 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0": { "em_relaxed": 0.24444444444444444, "em_relaxed_stderr": 0.03712537833614866, "em": 0.014814814814814815, "em_stderr": 0.010436494549594362, "em_conservative": 0.24444444444444444, "em_conservative_stderr": 0.03712537833614866 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0": { "em_relaxed": 0.1731266149870801, "em_relaxed_stderr": 0.01925784882518206, "em": 0.031007751937984496, "em_stderr": 0.00882270258926331, "em_conservative": 0.17054263565891473, "em_conservative_stderr": 0.019143434938568855 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0": { "em_relaxed": 0.06766917293233082, "em_relaxed_stderr": 0.015429701322264242, "em": 0.011278195488721804, "em_stderr": 0.0064868510822409155, "em_conservative": 0.06766917293233082, "em_conservative_stderr": 0.015429701322264242 }, "sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0": { "em_norm_relaxed": 0.15355086372360843, "em_norm_relaxed_stderr": 0.011173807886451477, "em_norm": 0.015355086372360844, "em_norm_stderr": 0.0038110143844746576, "em_conservative_norm": 0.118042226487524, "em_conservative_norm_stderr": 0.010000396690181516 }, "mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0": { "em_relaxed": 0.1276595744680851, "em_relaxed_stderr": 0.021815328994143183, "em": 0.05106382978723404, "em_stderr": 0.014390214367756187, "em_conservative": 0.1276595744680851, "em_conservative_stderr": 0.021815328994143183 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0": { "em_relaxed": 0.12121212121212122, "em_relaxed_stderr": 0.03296871474399962, "em": 0.030303030303030304, "em_stderr": 0.01731601731601732, "em_conservative": 0.12121212121212122, "em_conservative_stderr": 0.03296871474399962 }, "mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0": { "em_relaxed": 0.20245398773006135, "em_relaxed_stderr": 0.022289450459613447, "em": 0.06134969325153374, "em_stderr": 0.01331117614747688, "em_conservative": 0.20245398773006135, "em_conservative_stderr": 0.022289450459613447 }, "mmlu.prompt1_mqa_single_correct_sk:security_studies|0": { "em_relaxed": 0.1981981981981982, "em_relaxed_stderr": 0.026815582053277207, "em": 0.04504504504504504, "em_stderr": 0.01395143304433832, "em_conservative": 0.1981981981981982, "em_conservative_stderr": 0.026815582053277207 }, "skj1.prompt1_skj1_sk|0": { "em_relaxed": 0.02, "em_relaxed_stderr": 0.014070529413628957, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.02, "em_conservative_stderr": 0.014070529413628957 }, "mmlu.prompt1_mqa_single_correct_sk:college_physics|0": { "em_relaxed": 0.0784313725490196, "em_relaxed_stderr": 0.026751495257385273, "em": 0.00980392156862745, "em_stderr": 0.009803921568627454, "em_conservative": 0.0784313725490196, "em_conservative_stderr": 0.026751495257385273 }, "sklep.prompt3_hate_speech_sk:hate-speech|0": { "em_norm_relaxed": 0.6967399545109931, "em_norm_relaxed_stderr": 0.012661502663418697, "em_norm": 0.0, "em_norm_stderr": 0.0, "em_conservative_norm": 0.6967399545109931, "em_conservative_norm_stderr": 0.012661502663418697 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0": { "em_relaxed": 0.03759398496240601, "em_relaxed_stderr": 0.011684641953170118, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.03759398496240601, "em_conservative_stderr": 0.011684641953170118 }, "mmlu.prompt3_mqa_single_correct_sk:virology|0": { "em_relaxed": 0.1402439024390244, "em_relaxed_stderr": 0.027197928271392544, "em": 0.024390243902439025, "em_stderr": 0.012082375722391943, "em_conservative": 0.1402439024390244, "em_conservative_stderr": 0.027197928271392544 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0": { "em_relaxed": 0.05639097744360902, "em_relaxed_stderr": 0.014170263190664145, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.05639097744360902, "em_conservative_stderr": 0.014170263190664145 }, "mmlu.prompt2_mqa_single_correct_sk:security_studies|0": { "em_relaxed": 0.21621621621621623, "em_relaxed_stderr": 0.02769147769493752, "em": 0.04504504504504504, "em_stderr": 0.013951433044338322, "em_conservative": 0.21621621621621623, "em_conservative_stderr": 0.02769147769493752 }, "financial_exams.prompt3_completion_sk|0": { "em_relaxed": 0.24887556221889057, "em_relaxed_stderr": 0.01184218927916494, "em": 0.0037481259370314842, "em_stderr": 0.0016736960391157028, "em_conservative": 0.24887556221889057, "em_conservative_stderr": 0.01184218927916494 }, "mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0": { "em_relaxed": 0.18691588785046728, "em_relaxed_stderr": 0.03786499587281739, "em": 0.037383177570093455, "em_stderr": 0.01842518672106723, "em_conservative": 0.18691588785046728, "em_conservative_stderr": 0.03786499587281739 }, "senti_sk.prompt3_sentiment_3_class_sk|0": { "em_norm_relaxed": 0.40880906564036773, "em_norm_relaxed_stderr": 0.0071893047031815266, "em_norm": 0.10540945050245884, "em_norm_stderr": 0.004490706548553942, "em_conservative_norm": 0.26448578148385715, "em_conservative_norm_stderr": 0.006449996068936491 }, "mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0": { "em_relaxed": 0.1310344827586207, "em_relaxed_stderr": 0.028119838672931184, "em": 0.06896551724137931, "em_stderr": 0.021116290886061897, "em_conservative": 0.1310344827586207, "em_conservative_stderr": 0.028119838672931184 }, "sklep.prompt3_qa_sk:question-answering|0": { "exact_match_slk_prefix": 0.16143170197224252, "exact_match_slk_prefix_stderr": 0.0037586816998114083, "f1_slk": 0.3769392643579436, "f1_slk_stderr": 0.003773157275405368 }, "mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0": { "em_relaxed": 0.19630872483221476, "em_relaxed_stderr": 0.016283806947934177, "em": 0.018456375838926176, "em_stderr": 0.005517845461546564, "em_conservative": 0.19630872483221476, "em_conservative_stderr": 0.016283806947934177 }, "mmlu.prompt3_mqa_single_correct_sk:global_facts|0": { "em_relaxed": 0.12, "em_relaxed_stderr": 0.03265986323710906, "em": 0.01, "em_stderr": 0.009999999999999997, "em_conservative": 0.12, "em_conservative_stderr": 0.03265986323710906 }, "mmlu.prompt2_mqa_single_correct_sk:computer_security|0": { "em_relaxed": 0.11, "em_relaxed_stderr": 0.03144660377352203, "em": 0.02, "em_stderr": 0.014070529413628963, "em_conservative": 0.11, "em_conservative_stderr": 0.03144660377352203 }, "mmlu.prompt1_mqa_single_correct_sk:computer_security|0": { "em_relaxed": 0.08, "em_relaxed_stderr": 0.027265992434429082, "em": 0.02, "em_stderr": 0.014070529413628966, "em_conservative": 0.08, "em_conservative_stderr": 0.027265992434429082 }, "mmlu.prompt2_mqa_single_correct_sk:college_physics|0": { "em_relaxed": 0.12745098039215685, "em_relaxed_stderr": 0.03318224921942078, "em": 0.0392156862745098, "em_stderr": 0.01931444240436096, "em_conservative": 0.12745098039215685, "em_conservative_stderr": 0.03318224921942078 }, "globalmmlu-lite.prompt2_mqa_single_correct_sk|0": { "em_relaxed": 0.21, "em_relaxed_stderr": 0.020390916450326858, "em": 0.035, "em_stderr": 0.009200495791370004, "em_conservative": 0.21, "em_conservative_stderr": 0.020390916450326858 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0": { "em_relaxed": 0.26666666666666666, "em_relaxed_stderr": 0.03305282343736876, "em": 0.005555555555555556, "em_stderr": 0.005555555555555554, "em_conservative": 0.26666666666666666, "em_conservative_stderr": 0.03305282343736876 }, "mmlu.prompt3_mqa_single_correct_sk:anatomy|0": { "em_relaxed": 0.13432835820895522, "em_relaxed_stderr": 0.029568863219678807, "em": 0.029850746268656716, "em_stderr": 0.01475608176283378, "em_conservative": 0.13432835820895522, "em_conservative_stderr": 0.029568863219678807 }, "mmlu.prompt3_mqa_single_correct_sk:security_studies|0": { "em_relaxed": 0.15765765765765766, "em_relaxed_stderr": 0.024513522045494097, "em": 0.04954954954954955, "em_stderr": 0.014597835283794243, "em_conservative": 0.15765765765765766, "em_conservative_stderr": 0.024513522045494097 }, "movie_review.prompt3_sentiment_binary_sk|0": { "em_norm_relaxed": 0.11105577689243028, "em_norm_relaxed_stderr": 0.007013489433691037, "em_norm": 0.04731075697211155, "em_norm_stderr": 0.004738947450976647, "em_conservative_norm": 0.09412350597609562, "em_conservative_norm_stderr": 0.006517932474820501 }, "mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0": { "em_relaxed": 0.16, "em_relaxed_stderr": 0.03684529491774709, "em": 0.04, "em_stderr": 0.01969463855669323, "em_conservative": 0.16, "em_conservative_stderr": 0.03684529491774709 }, "mmlu.prompt2_mqa_single_correct_sk:astronomy|0": { "em_relaxed": 0.17763157894736842, "em_relaxed_stderr": 0.031103182383123398, "em": 0.05263157894736842, "em_stderr": 0.01817164224152954, "em_conservative": 0.17105263157894737, "em_conservative_stderr": 0.03064360707167709 }, "mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0": { "em_relaxed": 0.18, "em_relaxed_stderr": 0.038612291966536955, "em": 0.04, "em_stderr": 0.019694638556693223, "em_conservative": 0.18, "em_conservative_stderr": 0.038612291966536955 }, "mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0": { "em_relaxed": 0.05, "em_relaxed_stderr": 0.021904291355759026, "em": 0.01, "em_stderr": 0.009999999999999997, "em_conservative": 0.04, "em_conservative_stderr": 0.019694638556693237 }, "mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0": { "em_relaxed": 0.0896551724137931, "em_relaxed_stderr": 0.023807258824376967, "em": 0.041379310344827586, "em_stderr": 0.016597159859992713, "em_conservative": 0.0896551724137931, "em_conservative_stderr": 0.023807258824376967 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0": { "em_relaxed": 0.2777777777777778, "em_relaxed_stderr": 0.033477857599386325, "em": 0.005555555555555556, "em_stderr": 0.005555555555555546, "em_conservative": 0.2777777777777778, "em_conservative_stderr": 0.033477857599386325 }, "mmlu.prompt3_mqa_single_correct_sk:philosophy|0": { "em_relaxed": 0.237012987012987, "em_relaxed_stderr": 0.024270317456654856, "em": 0.05194805194805195, "em_stderr": 0.01266576753511598, "em_conservative": 0.237012987012987, "em_conservative_stderr": 0.024270317456654856 }, "mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0": { "em_relaxed": 0.21100917431192662, "em_relaxed_stderr": 0.039262212380368526, "em": 0.05504587155963303, "em_stderr": 0.0219460246091855, "em_conservative": 0.21100917431192662, "em_conservative_stderr": 0.039262212380368526 }, "mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0": { "em_relaxed": 0.01, "em_relaxed_stderr": 0.009999999999999997, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.01, "em_conservative_stderr": 0.009999999999999997 }, "mmlu.prompt3_mqa_single_correct_sk:world_religions|0": { "em_relaxed": 0.10526315789473684, "em_relaxed_stderr": 0.02353755765789256, "em": 0.04678362573099415, "em_stderr": 0.01619638995623562, "em_conservative": 0.10526315789473684, "em_conservative_stderr": 0.02353755765789256 }, "mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0": { "em_relaxed": 0.24324324324324326, "em_relaxed_stderr": 0.026710910573068247, "em": 0.04633204633204633, "em_stderr": 0.013086679986931563, "em_conservative": 0.23938223938223938, "em_conservative_stderr": 0.026565582359277095 }, "toxic_sk.prompt1_toxicity_sk|0": { "em_norm_relaxed": 0.45629965947786605, "em_norm_relaxed_stderr": 0.016790496508382532, "em_norm": 0.0, "em_norm_stderr": 0.0, "em_conservative_norm": 0.45629965947786605, "em_conservative_norm_stderr": 0.016790496508382532 }, "senti_sk.prompt1_sentiment_3_class_sk|0": { "em_norm_relaxed": 0.3200769724182168, "em_norm_relaxed_stderr": 0.0068221276139399335, "em_norm": 0.22642719692110327, "em_norm_stderr": 0.006120369470744044, "em_conservative_norm": 0.31922172332691895, "em_conservative_norm_stderr": 0.006817290681927466 }, "mmlu.prompt2_mqa_single_correct_sk:college_biology|0": { "em_relaxed": 0.19014084507042253, "em_relaxed_stderr": 0.03304706552420638, "em": 0.04225352112676056, "em_stderr": 0.01694131738100396, "em_conservative": 0.19014084507042253, "em_conservative_stderr": 0.03304706552420638 }, "mmlu.prompt3_mqa_single_correct_sk:public_relations|0": { "em_relaxed": 0.21100917431192662, "em_relaxed_stderr": 0.03926221238036853, "em": 0.07339449541284404, "em_stderr": 0.025093850407933094, "em_conservative": 0.21100917431192662, "em_conservative_stderr": 0.03926221238036853 }, "mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0": { "em_relaxed": 0.1969111969111969, "em_relaxed_stderr": 0.024757519994692216, "em": 0.03088803088803089, "em_stderr": 0.010771402065873048, "em_conservative": 0.1969111969111969, "em_conservative_stderr": 0.024757519994692216 }, "mmlu.prompt2_mqa_single_correct_sk:formal_logic|0": { "em_relaxed": 0.13043478260869565, "em_relaxed_stderr": 0.03154244565695702, "em": 0.008695652173913044, "em_stderr": 0.008695652173913031, "em_conservative": 0.08695652173913043, "em_conservative_stderr": 0.026390303420299282 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0": { "em_relaxed": 0.20202020202020202, "em_relaxed_stderr": 0.028606204289229872, "em": 0.07575757575757576, "em_stderr": 0.018852670234993107, "em_conservative": 0.20202020202020202, "em_conservative_stderr": 0.028606204289229872 }, "mmlu.prompt1_mqa_single_correct_sk:prehistory|0": { "em_relaxed": 0.15987460815047022, "em_relaxed_stderr": 0.02055172319966096, "em": 0.04075235109717868, "em_stderr": 0.011087359931149773, "em_conservative": 0.15987460815047022, "em_conservative_stderr": 0.02055172319966096 }, "mmlu.prompt1_mqa_single_correct_sk:econometrics|0": { "em_relaxed": 0.125, "em_relaxed_stderr": 0.03139045014587016, "em": 0.026785714285714284, "em_stderr": 0.015324784082343043, "em_conservative": 0.125, "em_conservative_stderr": 0.03139045014587016 }, "arc.prompt1_mqa_single_correct_sk|0": { "em_relaxed": 0.1710863986313088, "em_relaxed_stderr": 0.011018968811092407, "em": 0.013686911890504704, "em_stderr": 0.0033996852610928244, "em_conservative": 0.1710863986313088, "em_conservative_stderr": 0.011018968811092407 }, "mmlu.prompt1_mqa_single_correct_sk:business_ethics|0": { "em_relaxed": 0.14, "em_relaxed_stderr": 0.03487350880197772, "em": 0.02, "em_stderr": 0.014070529413628957, "em_conservative": 0.14, "em_conservative_stderr": 0.03487350880197772 }, "mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0": { "em_relaxed": 0.14018691588785046, "em_relaxed_stderr": 0.03372118539369118, "em": 0.056074766355140186, "em_stderr": 0.02234599008672909, "em_conservative": 0.14018691588785046, "em_conservative_stderr": 0.03372118539369118 }, "truthfulqa.prompt2_mqa_single_correct_sk:mc1|0": { "em_relaxed": 0.016709511568123392, "em_relaxed_stderr": 0.004598458668423289, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.016709511568123392, "em_conservative_stderr": 0.004598458668423289 }, "mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0": { "em_relaxed": 0.14505776636713735, "em_relaxed_stderr": 0.01262551435912769, "em": 0.03209242618741977, "em_stderr": 0.006318709962464618, "em_conservative": 0.14505776636713735, "em_conservative_stderr": 0.01262551435912769 }, "mmlu.prompt1_mqa_single_correct_sk:astronomy|0": { "em_relaxed": 0.14473684210526316, "em_relaxed_stderr": 0.028631951845930412, "em": 0.03289473684210526, "em_stderr": 0.014514818445457586, "em_conservative": 0.13815789473684212, "em_conservative_stderr": 0.028081042939576552 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0": { "em_relaxed": 0.21481481481481482, "em_relaxed_stderr": 0.03547854198560824, "em": 0.02962962962962963, "em_stderr": 0.014648038602753806, "em_conservative": 0.2074074074074074, "em_conservative_stderr": 0.03502553170678318 }, "senti_sk.prompt2_sentiment_3_class_sk|0": { "em_norm_relaxed": 0.383365405174257, "em_norm_relaxed_stderr": 0.007110221850748201, "em_norm": 0.019029292281376953, "em_norm_stderr": 0.001998030377272694, "em_conservative_norm": 0.37716484926234767, "em_conservative_norm_stderr": 0.007087856422819165 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0": { "em_relaxed": 0.23703703703703705, "em_relaxed_stderr": 0.03673731683969506, "em": 0.022222222222222223, "em_stderr": 0.012733899715059683, "em_conservative": 0.23703703703703705, "em_conservative_stderr": 0.03673731683969506 }, "mmlu.prompt3_mqa_single_correct_sk:formal_logic|0": { "em_relaxed": 0.11304347826086956, "em_relaxed_stderr": 0.029656609316837397, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.06956521739130435, "em_conservative_stderr": 0.02382794649967283 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0": { "em_relaxed": 0.30569948186528495, "em_relaxed_stderr": 0.033248379397581594, "em": 0.046632124352331605, "em_stderr": 0.015216761819262604, "em_conservative": 0.3005181347150259, "em_conservative_stderr": 0.0330881859441575 }, "sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0": { "em_norm_relaxed": 0.3512476007677543, "em_norm_relaxed_stderr": 0.014795198864354053, "em_norm": 0.0009596928982725527, "em_norm_stderr": 0.0009596928982725573, "em_conservative_norm": 0.34836852207293667, "em_conservative_norm_stderr": 0.014767096602802959 }, "sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0": { "em_norm_relaxed": 0.8809980806142035, "em_norm_relaxed_stderr": 0.010035501945449067, "em_norm": 0.44817658349328215, "em_norm_stderr": 0.015413429449614385, "em_conservative_norm": 0.8800383877159309, "em_conservative_norm_stderr": 0.010070396971355563 }, "skj1.prompt2_skj1_sk|0": { "em_relaxed": 0.06, "em_relaxed_stderr": 0.023868325657594166, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.05, "em_conservative_stderr": 0.021904291355759067 }, "mmlu.prompt2_mqa_single_correct_sk:international_law|0": { "em_relaxed": 0.16101694915254236, "em_relaxed_stderr": 0.03397969368949715, "em": 0.05084745762711865, "em_stderr": 0.020309989475094187, "em_conservative": 0.16101694915254236, "em_conservative_stderr": 0.03397969368949715 }, "sklep.prompt1_nli_binary_sk:rte|0": { "em_norm_relaxed": 0.030120481927710843, "em_norm_relaxed_stderr": 0.004196301257202765, "em_norm": 0.0006024096385542169, "em_norm_stderr": 0.0006024096385542043, "em_conservative_norm": 0.030120481927710843, "em_conservative_norm_stderr": 0.004196301257202765 }, "mmlu.prompt3_mqa_single_correct_sk:human_aging|0": { "em_relaxed": 0.1278538812785388, "em_relaxed_stderr": 0.02261639276396452, "em": 0.0684931506849315, "em_stderr": 0.017107571020219335, "em_conservative": 0.1278538812785388, "em_conservative_stderr": 0.02261639276396452 }, "mmlu.prompt3_mqa_single_correct_sk:management|0": { "em_relaxed": 0.2, "em_relaxed_stderr": 0.040201512610368445, "em": 0.06, "em_stderr": 0.023868325657594162, "em_conservative": 0.2, "em_conservative_stderr": 0.040201512610368445 }, "mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0": { "em_relaxed": 0.15217391304347827, "em_relaxed_stderr": 0.02165993479655374, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.14130434782608695, "em_conservative_stderr": 0.02100540442245809 }, "truthfulqa.prompt1_mqa_single_correct_sk:mc1|0": { "em_relaxed": 0.007712082262210797, "em_relaxed_stderr": 0.003138297919968842, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.007712082262210797, "em_conservative_stderr": 0.003138297919968842 }, "senti_sk.prompt1_sentiment_binary_sk|0": { "em_norm_relaxed": 0.31223175965665234, "em_norm_relaxed_stderr": 0.0075906663188793026, "em_norm": 0.1968884120171674, "em_norm_stderr": 0.006513558648330005, "em_conservative_norm": 0.3111587982832618, "em_conservative_norm_stderr": 0.007583521183648285 }, "mmlu.prompt3_mqa_single_correct_sk:astronomy|0": { "em_relaxed": 0.1513157894736842, "em_relaxed_stderr": 0.029162631596843985, "em": 0.019736842105263157, "em_stderr": 0.011319355505772411, "em_conservative": 0.14473684210526316, "em_conservative_stderr": 0.028631951845930412 }, "mmlu.prompt2_mqa_single_correct_sk:human_aging|0": { "em_relaxed": 0.1735159817351598, "em_relaxed_stderr": 0.02564830822412722, "em": 0.0867579908675799, "em_stderr": 0.01906423736413682, "em_conservative": 0.1735159817351598, "em_conservative_stderr": 0.02564830822412722 }, "mmlu.prompt1_mqa_single_correct_sk:formal_logic|0": { "em_relaxed": 0.12173913043478261, "em_relaxed_stderr": 0.030624883036764184, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.0782608695652174, "em_conservative_stderr": 0.025154976812194616 }, "mmlu.prompt3_mqa_single_correct_sk:sociology|0": { "em_relaxed": 0.3350253807106599, "em_relaxed_stderr": 0.03371423842235921, "em": 0.05583756345177665, "em_stderr": 0.016400555297895067, "em_conservative": 0.3350253807106599, "em_conservative_stderr": 0.03371423842235921 }, "mmlu.prompt2_mqa_single_correct_sk:econometrics|0": { "em_relaxed": 0.11607142857142858, "em_relaxed_stderr": 0.03040253249894973, "em": 0.017857142857142856, "em_stderr": 0.012569900164490047, "em_conservative": 0.11607142857142858, "em_conservative_stderr": 0.03040253249894973 }, "mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0": { "em_relaxed": 0.08847184986595175, "em_relaxed_stderr": 0.014723671750091158, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.08579088471849866, "em_conservative_stderr": 0.01452017587959949 }, "mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0": { "em_relaxed": 0.06060606060606061, "em_relaxed_stderr": 0.024102876029567216, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.050505050505050504, "em_conservative_stderr": 0.02212079324207198 }, "mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0": { "em_relaxed": 0.20134228187919462, "em_relaxed_stderr": 0.016439528475054862, "em": 0.03355704697986577, "em_stderr": 0.0073828066874755535, "em_conservative": 0.20134228187919462, "em_conservative_stderr": 0.016439528475054862 }, "mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0": { "em_relaxed": 0.09090909090909091, "em_relaxed_stderr": 0.02903984386363362, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.08080808080808081, "em_conservative_stderr": 0.027530712883605948 }, "globalmmlu-lite.prompt3_mqa_single_correct_sk|0": { "em_relaxed": 0.2025, "em_relaxed_stderr": 0.02011830610666578, "em": 0.025, "em_stderr": 0.007816023641463063, "em_conservative": 0.2025, "em_conservative_stderr": 0.02011830610666578 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0": { "em_relaxed": 0.15762273901808785, "em_relaxed_stderr": 0.018546804758294254, "em": 0.01808785529715762, "em_stderr": 0.006783224657656192, "em_conservative": 0.14987080103359174, "em_conservative_stderr": 0.018168008667456638 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0": { "em_relaxed": 0.17050691244239632, "em_relaxed_stderr": 0.025588834483805927, "em": 0.013824884792626729, "em_stderr": 0.007944762237164815, "em_conservative": 0.16589861751152074, "em_conservative_stderr": 0.025310686717794278 }, "mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0": { "em_relaxed": 0.1559633027522936, "em_relaxed_stderr": 0.03491242362801975, "em": 0.05504587155963303, "em_stderr": 0.021946024609185506, "em_conservative": 0.1559633027522936, "em_conservative_stderr": 0.03491242362801975 }, "mmlu.prompt3_mqa_single_correct_sk:computer_security|0": { "em_relaxed": 0.13, "em_relaxed_stderr": 0.03379976689896309, "em": 0.02, "em_stderr": 0.014070529413628963, "em_conservative": 0.13, "em_conservative_stderr": 0.03379976689896309 }, "toxic_sk.prompt2_toxicity_sk|0": { "em_norm_relaxed": 0.47559591373439275, "em_norm_relaxed_stderr": 0.016834908286515442, "em_norm": 0.0, "em_norm_stderr": 0.0, "em_conservative_norm": 0.47559591373439275, "em_conservative_norm_stderr": 0.016834908286515442 }, "multi_wiki_qa.prompt3_qa_sk|0": { "exact_match_slk_prefix": 0.34000400240144085, "exact_match_slk_prefix_stderr": 0.006701954125609285, "f1_slk": 0.5050071786470199, "f1_slk_stderr": 0.005797838571943781 }, "financial_exams.prompt1_completion_sk|0": { "em_relaxed": 0.20089955022488756, "em_relaxed_stderr": 0.010974256206370726, "em": 0.009745127436281859, "em_stderr": 0.002690618851324012, "em_conservative": 0.20089955022488756, "em_conservative_stderr": 0.010974256206370726 }, "mmlu.prompt1_mqa_single_correct_sk:philosophy|0": { "em_relaxed": 0.21103896103896103, "em_relaxed_stderr": 0.023288413887090093, "em": 0.06818181818181818, "em_stderr": 0.014385689805382164, "em_conservative": 0.21103896103896103, "em_conservative_stderr": 0.023288413887090093 }, "mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0": { "em_relaxed": 0.2147239263803681, "em_relaxed_stderr": 0.022777692313730433, "em": 0.06748466257668712, "em_stderr": 0.013915180741090785, "em_conservative": 0.2116564417177914, "em_conservative_stderr": 0.02265853497858724 }, "mmlu.prompt1_mqa_single_correct_sk:marketing|0": { "em_relaxed": 0.15384615384615385, "em_relaxed_stderr": 0.023636873317489277, "em": 0.03418803418803419, "em_stderr": 0.011904341997629792, "em_conservative": 0.15384615384615385, "em_conservative_stderr": 0.023636873317489277 }, "globalmmlu-lite.prompt1_mqa_single_correct_sk|0": { "em_relaxed": 0.1825, "em_relaxed_stderr": 0.01933698975881026, "em": 0.025, "em_stderr": 0.007816023641463065, "em_conservative": 0.1825, "em_conservative_stderr": 0.01933698975881026 }, "mmlu.prompt1_mqa_single_correct_sk:professional_law|0": { "em_relaxed": 0.12724167378309137, "em_relaxed_stderr": 0.009742462799429212, "em": 0.0017079419299743809, "em_stderr": 0.0012071811002646103, "em_conservative": 0.12724167378309137, "em_conservative_stderr": 0.009742462799429212 }, "mmlu.prompt2_mqa_single_correct_sk:machine_learning|0": { "em_relaxed": 0.14285714285714285, "em_relaxed_stderr": 0.033213611069662675, "em": 0.026785714285714284, "em_stderr": 0.015324784082343059, "em_conservative": 0.125, "em_conservative_stderr": 0.03139045014587016 }, "mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0": { "em_relaxed": 0.2727272727272727, "em_relaxed_stderr": 0.044988332664126655, "em": 0.050505050505050504, "em_stderr": 0.022120793242071984, "em_conservative": 0.26262626262626265, "em_conservative_stderr": 0.044452876769839444 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0": { "em_relaxed": 0.30569948186528495, "em_relaxed_stderr": 0.03324837939758159, "em": 0.025906735751295335, "em_stderr": 0.011464523356953171, "em_conservative": 0.30569948186528495, "em_conservative_stderr": 0.03324837939758159 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0": { "em_relaxed": 0.17333333333333334, "em_relaxed_stderr": 0.031010810941145068, "em": 0.02666666666666667, "em_stderr": 0.013198422611557013, "em_conservative": 0.17333333333333334, "em_conservative_stderr": 0.031010810941145068 }, "mmlu.prompt1_mqa_single_correct_sk:management|0": { "em_relaxed": 0.15, "em_relaxed_stderr": 0.0358870281282637, "em": 0.02, "em_stderr": 0.01407052941362896, "em_conservative": 0.15, "em_conservative_stderr": 0.0358870281282637 }, "mmlu.prompt1_mqa_single_correct_sk:sociology|0": { "em_relaxed": 0.3096446700507614, "em_relaxed_stderr": 0.03302479343505637, "em": 0.050761421319796954, "em_stderr": 0.015679295366526198, "em_conservative": 0.3096446700507614, "em_conservative_stderr": 0.03302479343505637 }, "mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0": { "em_relaxed": 0.23232323232323232, "em_relaxed_stderr": 0.04266016017054687, "em": 0.030303030303030304, "em_stderr": 0.01731601731601731, "em_conservative": 0.2222222222222222, "em_conservative_stderr": 0.041996052556580815 }, "skj1.prompt3_skj1_sk|0": { "em_relaxed": 0.05, "em_relaxed_stderr": 0.021904291355759067, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.03, "em_conservative_stderr": 0.017144660799776543 }, "movie_review.prompt1_sentiment_binary_sk|0": { "em_norm_relaxed": 0.6329681274900398, "em_norm_relaxed_stderr": 0.010758931054934645, "em_norm": 0.38745019920318724, "em_norm_stderr": 0.010874391376667836, "em_conservative_norm": 0.6324701195219123, "em_conservative_norm_stderr": 0.010761991548945318 }, "mmlu.prompt1_mqa_single_correct_sk:anatomy|0": { "em_relaxed": 0.12686567164179105, "em_relaxed_stderr": 0.028859365115889928, "em": 0.022388059701492536, "em_stderr": 0.012828198053329663, "em_conservative": 0.12686567164179105, "em_conservative_stderr": 0.028859365115889928 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0": { "em_relaxed": 0.10101010101010101, "em_relaxed_stderr": 0.030440148787487714, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.10101010101010101, "em_conservative_stderr": 0.030440148787487714 }, "mmlu.prompt2_mqa_single_correct_sk:philosophy|0": { "em_relaxed": 0.2922077922077922, "em_relaxed_stderr": 0.02595551182455085, "em": 0.10064935064935066, "em_stderr": 0.017171207138792156, "em_conservative": 0.2922077922077922, "em_conservative_stderr": 0.02595551182455085 }, "mmlu.prompt1_mqa_single_correct_sk:human_aging|0": { "em_relaxed": 0.1187214611872146, "em_relaxed_stderr": 0.021907507389103187, "em": 0.0365296803652968, "em_stderr": 0.012706146532849703, "em_conservative": 0.1187214611872146, "em_conservative_stderr": 0.021907507389103187 }, "mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0": { "em_relaxed": 0.10187667560321716, "em_relaxed_stderr": 0.015683171805464465, "em": 0.0160857908847185, "em_stderr": 0.006522714286001283, "em_conservative": 0.09651474530831099, "em_conservative_stderr": 0.015310377502350168 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0": { "em_relaxed": 0.13131313131313133, "em_relaxed_stderr": 0.034117144062155554, "em": 0.010101010101010102, "em_stderr": 0.010101010101010109, "em_conservative": 0.12121212121212122, "em_conservative_stderr": 0.03296871474399963 }, "mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0": { "em_relaxed": 0.13257575757575757, "em_relaxed_stderr": 0.020910766968363862, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.13257575757575757, "em_conservative_stderr": 0.020910766968363862 }, "mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0": { "em_relaxed": 0.21308724832214765, "em_relaxed_stderr": 0.01678740388759121, "em": 0.050335570469798654, "em_stderr": 0.008963220911107225, "em_conservative": 0.21308724832214765, "em_conservative_stderr": 0.01678740388759121 }, "mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0": { "em_relaxed": 0.12413793103448276, "em_relaxed_stderr": 0.0274782369836366, "em": 0.041379310344827586, "em_stderr": 0.016597159859992713, "em_conservative": 0.12413793103448276, "em_conservative_stderr": 0.0274782369836366 }, "mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0": { "em_relaxed": 0.25316455696202533, "em_relaxed_stderr": 0.0347027655481443, "em": 0.05063291139240506, "em_stderr": 0.017497810541168043, "em_conservative": 0.25316455696202533, "em_conservative_stderr": 0.0347027655481443 }, "sklep.prompt1_hate_speech_sk:hate-speech|0": { "em_norm_relaxed": 0.6921910538286581, "em_norm_relaxed_stderr": 0.012714401009923647, "em_norm": 0.0, "em_norm_stderr": 0.0, "em_conservative_norm": 0.6921910538286581, "em_conservative_norm_stderr": 0.012714401009923647 }, "mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0": { "em_relaxed": 0.2092426187419769, "em_relaxed_stderr": 0.014583336673745648, "em": 0.0693196405648267, "em_stderr": 0.009106230246473392, "em_conservative": 0.2092426187419769, "em_conservative_stderr": 0.014583336673745648 }, "mmlu.prompt2_mqa_single_correct_sk:world_religions|0": { "em_relaxed": 0.0935672514619883, "em_relaxed_stderr": 0.022335993231163274, "em": 0.04678362573099415, "em_stderr": 0.016196389956235627, "em_conservative": 0.0935672514619883, "em_conservative_stderr": 0.022335993231163274 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0": { "em_relaxed": 0.18439716312056736, "em_relaxed_stderr": 0.023134670670309505, "em": 0.02127659574468085, "em_stderr": 0.00860850820920177, "em_conservative": 0.1773049645390071, "em_conservative_stderr": 0.022783829728850895 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0": { "em_relaxed": 0.2777777777777778, "em_relaxed_stderr": 0.03347785759938633, "em": 0.005555555555555556, "em_stderr": 0.005555555555555554, "em_conservative": 0.2777777777777778, "em_conservative_stderr": 0.03347785759938633 }, "mmlu.prompt3_mqa_single_correct_sk:college_biology|0": { "em_relaxed": 0.176056338028169, "em_relaxed_stderr": 0.03207487957537854, "em": 0.035211267605633804, "em_stderr": 0.015521989489917001, "em_conservative": 0.176056338028169, "em_conservative_stderr": 0.03207487957537854 }, "mmlu.prompt2_mqa_single_correct_sk:global_facts|0": { "em_relaxed": 0.18, "em_relaxed_stderr": 0.03861229196653694, "em": 0.13, "em_stderr": 0.03379976689896309, "em_conservative": 0.18, "em_conservative_stderr": 0.03861229196653694 }, "mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0": { "em_relaxed": 0.25766871165644173, "em_relaxed_stderr": 0.0242598463239315, "em": 0.08895705521472393, "em_stderr": 0.015791302137788298, "em_conservative": 0.25766871165644173, "em_conservative_stderr": 0.0242598463239315 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0": { "em_relaxed": 0.15656565656565657, "em_relaxed_stderr": 0.025890520358141454, "em": 0.030303030303030304, "em_stderr": 0.012213156893572809, "em_conservative": 0.15656565656565657, "em_conservative_stderr": 0.025890520358141454 }, "mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0": { "em_relaxed": 0.0026702269692923898, "em_relaxed_stderr": 0.001886873052453708, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.0026702269692923898, "em_conservative_stderr": 0.001886873052453708 }, "mmlu.prompt3_mqa_single_correct_sk:college_physics|0": { "em_relaxed": 0.12745098039215685, "em_relaxed_stderr": 0.03318224921942076, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.12745098039215685, "em_conservative_stderr": 0.03318224921942076 }, "mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0": { "em_relaxed": 0.09090909090909091, "em_relaxed_stderr": 0.01772676309015986, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.09090909090909091, "em_conservative_stderr": 0.01772676309015986 }, "mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0": { "em_relaxed": 0.13924050632911392, "em_relaxed_stderr": 0.027629554439252756, "em": 0.02531645569620253, "em_stderr": 0.012536706047327604, "em_conservative": 0.13924050632911392, "em_conservative_stderr": 0.027629554439252756 }, "sklep.prompt3_nli_binary_sk:rte|0": { "em_norm_relaxed": 0.2319277108433735, "em_norm_relaxed_stderr": 0.010362252629508847, "em_norm": 0.0012048192771084338, "em_norm_stderr": 0.0008516790803748009, "em_conservative_norm": 0.2319277108433735, "em_conservative_norm_stderr": 0.010362252629508847 }, "slovakcopa.prompt2_copa_sk|0": { "em_relaxed": 0.51, "em_relaxed_stderr": 0.02237859698923078, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.326, "em_conservative_stderr": 0.020984009562393567 }, "mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0": { "em_relaxed": 0.19626168224299065, "em_relaxed_stderr": 0.038576441428227824, "em": 0.056074766355140186, "em_stderr": 0.02234599008672909, "em_conservative": 0.19626168224299065, "em_conservative_stderr": 0.038576441428227824 }, "multi_wiki_qa.prompt1_qa_sk|0": { "exact_match_slk_prefix": 0.36401841104662797, "exact_match_slk_prefix_stderr": 0.00680726616133214, "f1_slk": 0.5156574509584114, "f1_slk_stderr": 0.0058319674487835125 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0": { "em_relaxed": 0.16791044776119404, "em_relaxed_stderr": 0.016160210122502162, "em": 0.016791044776119403, "em_stderr": 0.005555010467354857, "em_conservative": 0.16791044776119404, "em_conservative_stderr": 0.016160210122502162 }, "mmlu.prompt1_mqa_single_correct_sk:global_facts|0": { "em_relaxed": 0.11, "em_relaxed_stderr": 0.031446603773522035, "em": 0.06, "em_stderr": 0.02386832565759416, "em_conservative": 0.11, "em_conservative_stderr": 0.031446603773522035 }, "mmlu.prompt2_mqa_single_correct_sk:sociology|0": { "em_relaxed": 0.3553299492385787, "em_relaxed_stderr": 0.034186655739878676, "em": 0.07106598984771574, "em_stderr": 0.018352502878671184, "em_conservative": 0.3553299492385787, "em_conservative_stderr": 0.034186655739878676 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0": { "em_relaxed": 0.1950354609929078, "em_relaxed_stderr": 0.02363698094391562, "em": 0.02127659574468085, "em_stderr": 0.00860850820920177, "em_conservative": 0.1879432624113475, "em_conservative_stderr": 0.023305230769714243 }, "mmlu.prompt1_mqa_single_correct_sk:public_relations|0": { "em_relaxed": 0.1743119266055046, "em_relaxed_stderr": 0.03650561586028542, "em": 0.027522935779816515, "em_stderr": 0.015742553732396008, "em_conservative": 0.1743119266055046, "em_conservative_stderr": 0.03650561586028542 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0": { "em_relaxed": 0.18691588785046728, "em_relaxed_stderr": 0.026711670245826564, "em": 0.004672897196261682, "em_stderr": 0.004672897196261678, "em_conservative": 0.17757009345794392, "em_conservative_stderr": 0.02618451638732211 }, "mmlu.prompt2_mqa_single_correct_sk:anatomy|0": { "em_relaxed": 0.1791044776119403, "em_relaxed_stderr": 0.03324844546213894, "em": 0.05970149253731343, "em_stderr": 0.020544692585423446, "em_conservative": 0.1791044776119403, "em_conservative_stderr": 0.03324844546213894 }, "mmlu.prompt2_mqa_single_correct_sk:marketing|0": { "em_relaxed": 0.2094017094017094, "em_relaxed_stderr": 0.02665569965392276, "em": 0.0811965811965812, "em_stderr": 0.017893784904018422, "em_conservative": 0.20512820512820512, "em_conservative_stderr": 0.026453508054040356 }, "mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0": { "em_relaxed": 0.14492753623188406, "em_relaxed_stderr": 0.02122807279169736, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.13043478260869565, "em_conservative_stderr": 0.020308668108561446 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0": { "em_relaxed": 0.22727272727272727, "em_relaxed_stderr": 0.02985751567338641, "em": 0.0707070707070707, "em_stderr": 0.01826310542019951, "em_conservative": 0.22727272727272727, "em_conservative_stderr": 0.02985751567338641 }, "slovakcopa.prompt1_copa_sk|0": { "em_relaxed": 0.618, "em_relaxed_stderr": 0.021750820591250834, "em": 0.0, "em_stderr": 0.0, "em_conservative": 0.508, "em_conservative_stderr": 0.022380208834928028 }, "mmlu.prompt2_mqa_single_correct_sk:nutrition|0": { "em_relaxed": 0.19801980198019803, "em_relaxed_stderr": 0.02293151046482573, "em": 0.0297029702970297, "em_stderr": 0.00976897099052845, "em_conservative": 0.19801980198019803, "em_conservative_stderr": 0.02293151046482573 }, "mmlu.prompt3_mqa_single_correct_sk:machine_learning|0": { "em_relaxed": 0.125, "em_relaxed_stderr": 0.03139045014587016, "em": 0.008928571428571428, "em_stderr": 0.008928571428571423, "em_conservative": 0.11607142857142858, "em_conservative_stderr": 0.03040253249894973 }, "mmlu.prompt3_mqa_single_correct_sk:_average|0": { "em_relaxed": 0.15750731608871743, "em_relaxed_stderr": 0.027457423641973053, "em": 0.028327141879067475, "em_stderr": 0.010791129591908543, "em_conservative": 0.15358971887332207, "em_conservative_stderr": 0.027071657841089895 }, "mmlu.prompt1_mqa_single_correct_sk:_average|0": { "em_relaxed": 0.1431281879704793, "em_relaxed_stderr": 0.02601762436348415, "em": 0.023388116537651122, "em_stderr": 0.010092739621009001, "em_conservative": 0.14035369812713153, "em_conservative_stderr": 0.02572013192933555 }, "mmlu.prompt2_mqa_single_correct_sk:_average|0": { "em_relaxed": 0.1796468718485245, "em_relaxed_stderr": 0.02870969018742619, "em": 0.04274500178923145, "em_stderr": 0.01407239223594206, "em_conservative": 0.17602310250998576, "em_conservative_stderr": 0.028335799048817806 }, "all": { "em_relaxed": 0.16276835973658582, "em_relaxed_stderr": 0.02594967705725322, "em": 0.029667227870516465, "em_stderr": 0.010711849450086212, "em_conservative": 0.15781564927252603, "em_conservative_stderr": 0.025592515877326957, "exact_match_slk_prefix": 0.25920345417449087, "exact_match_slk_prefix_stderr": 0.005273401550208795, "f1_slk": 0.4424731081623923, "f1_slk_stderr": 0.004828570063081536, "em_norm_relaxed": 0.37655055755239747, "em_norm_relaxed_stderr": 0.009574346535835498, "em_norm": 0.06270134241081571, "em_norm_stderr": 0.0027847027649703544, "em_conservative_norm": 0.3652434918007519, "em_conservative_norm_stderr": 0.009448048478463963 } }, "versions": {}, "config_tasks": { "mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:miscellaneous", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "financial_exams.prompt2_completion_sk|0": { "name": "financial_exams.prompt2_completion_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/slovak-financial-exam", "hf_subset": "default", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "7cdbfe635c3ed0d8b4df2ee04429d8afa651b33c", "hf_filter": null, "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:machine_learning|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:machine_learning", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "belebele.prompt3_mqa_single_correct_sk|0": { "name": "belebele.prompt3_mqa_single_correct_sk", "prompt_function": "prompt_fn", "hf_repo": "facebook/belebele", "hf_subset": "slk_Latn", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2", "hf_filter": null, "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:professional_law|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:professional_law", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:college_computer_science", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:professional_accounting", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_us_history", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_us_history", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:professional_law|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:professional_law", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt1_qa_sk:question-answering|0": { "name": "sklep.prompt1_qa_sk:question-answering", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "question-answering", "metrics": [ { "metric_name": "exact_match_slk_prefix", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "f1_slk", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 126, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:college_chemistry", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:college_mathematics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_physics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_statistics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:professional_medicine", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "slovakcopa.prompt3_copa_sk|0": { "name": "slovakcopa.prompt3_copa_sk", "prompt_function": "prompt_fn", "hf_repo": "NaiveNeuron/SlovakCOPA", "hf_subset": "default", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "5c61e4cd9343f9b4fab623bdc50bb895cc170771", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "validation", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "movie_review.prompt2_sentiment_binary_sk|0": { "name": "movie_review.prompt2_sentiment_binary_sk", "prompt_function": "prompt_fn", "hf_repo": "mteb/slovak_movie_review_sentiment", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "29a7405aabcfd4860a51ae6f65a5650d63108f26", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt2_qa_sk:question-answering|0": { "name": "sklep.prompt2_qa_sk:question-answering", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "question-answering", "metrics": [ { "metric_name": "exact_match_slk_prefix", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "f1_slk", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 126, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:college_biology|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:college_biology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:nutrition|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:nutrition", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_biology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "arc.prompt3_mqa_single_correct_sk|0": { "name": "arc.prompt3_mqa_single_correct_sk", "prompt_function": "prompt_fn", "hf_repo": "jon-tow/okapi_arc_challenge", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": [ "solve" ], "scorer": "score", "sample_fields": "record_to_sample", "sample_to_fewshot": null, "filter": null, "hf_revision": "56aa31aedeb55f0635cc7c3c13f7f86b276f9a4e", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:econometrics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:econometrics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:prehistory|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:prehistory", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "toxic_sk.prompt3_toxicity_sk|0": { "name": "toxic_sk.prompt3_toxicity_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/toxic-sk", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "2fb416e18785c128e7ec934c07a14c803a88cdb3", "hf_filter": null, "hf_avail_splits": [ "train", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "belebele.prompt1_mqa_single_correct_sk|0": { "name": "belebele.prompt1_mqa_single_correct_sk", "prompt_function": "prompt_fn", "hf_repo": "facebook/belebele", "hf_subset": "slk_Latn", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2", "hf_filter": null, "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "arc.prompt2_mqa_single_correct_sk|0": { "name": "arc.prompt2_mqa_single_correct_sk", "prompt_function": "prompt_fn", "hf_repo": "jon-tow/okapi_arc_challenge", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": [ "solve" ], "scorer": "score", "sample_fields": "record_to_sample", "sample_to_fewshot": null, "filter": null, "hf_revision": "56aa31aedeb55f0635cc7c3c13f7f86b276f9a4e", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "multi_wiki_qa.prompt2_qa_sk|0": { "name": "multi_wiki_qa.prompt2_qa_sk", "prompt_function": "prompt_fn", "hf_repo": "alexandrainst/multi-wiki-qa", "hf_subset": "sk", "metrics": [ { "metric_name": "exact_match_slk_prefix", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "f1_slk", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "9bdb3350487e399bd1cd873e1395290d636db5ce", "hf_filter": "", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 126, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:management|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:management", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:abstract_algebra", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:college_chemistry", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt3_nli_3_classes_sk:nli|0": { "name": "sklep.prompt3_nli_3_classes_sk:nli", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "nli", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:college_medicine|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:college_medicine", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:business_ethics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:business_ethics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:college_computer_science", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:international_law|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:international_law", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:international_law|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:international_law", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "truthfulqa.prompt3_mqa_single_correct_sk:mc1|0": { "name": "truthfulqa.prompt3_mqa_single_correct_sk:mc1", "prompt_function": "prompt_fn", "hf_repo": "alexandrainst/m_truthfulqa", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "f0445d470f1925882b990f5f247fdcf288972f60", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "val" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:prehistory|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:prehistory", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:nutrition|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:nutrition", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:conceptual_physics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:college_computer_science", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:college_medicine|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:college_medicine", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:college_medicine|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:college_medicine", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_psychology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt2_nli_binary_sk:rte|0": { "name": "sklep.prompt2_nli_binary_sk:rte", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "rte", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt2_hate_speech_sk:hate-speech|0": { "name": "sklep.prompt2_hate_speech_sk:hate-speech", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "hate-speech", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:human_sexuality", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_psychology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_us_history", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:marketing|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:marketing", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:logical_fallacies", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:conceptual_physics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:virology|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:virology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:medical_genetics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:moral_scenarios", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:world_religions|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:world_religions", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "senti_sk.prompt3_sentiment_binary_sk|0": { "name": "senti_sk.prompt3_sentiment_binary_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/senti-sk", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d", "hf_filter": "", "hf_avail_splits": [ "train", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_physics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt1_nli_3_classes_sk:nli|0": { "name": "sklep.prompt1_nli_3_classes_sk:nli", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "nli", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:moral_scenarios", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:business_ethics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:business_ethics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:public_relations|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:public_relations", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_statistics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "senti_sk.prompt2_sentiment_binary_sk|0": { "name": "senti_sk.prompt2_sentiment_binary_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/senti-sk", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d", "hf_filter": "", "hf_avail_splits": [ "train", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:medical_genetics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "belebele.prompt2_mqa_single_correct_sk|0": { "name": "belebele.prompt2_mqa_single_correct_sk", "prompt_function": "prompt_fn", "hf_repo": "facebook/belebele", "hf_subset": "slk_Latn", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2", "hf_filter": null, "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:virology|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:virology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt2_nli_3_classes_sk:nli|0": { "name": "sklep.prompt2_nli_3_classes_sk:nli", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "nli", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_european_history", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0": { "name": "sklep.prompt3_sentiment_binary_sk:sentiment-analysis", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "sentiment-analysis", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:conceptual_physics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:moral_disputes", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:security_studies|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:security_studies", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "skj1.prompt1_skj1_sk|0": { "name": "skj1.prompt1_skj1_sk", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/SKJ1", "hf_subset": "default", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "e615c60b415389b629703f82d3a79810dbde6734", "hf_filter": null, "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:college_physics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:college_physics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt3_hate_speech_sk:hate-speech|0": { "name": "sklep.prompt3_hate_speech_sk:hate-speech", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "hate-speech", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:virology|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:virology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:security_studies|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:security_studies", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "financial_exams.prompt3_completion_sk|0": { "name": "financial_exams.prompt3_completion_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/slovak-financial-exam", "hf_subset": "default", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "7cdbfe635c3ed0d8b4df2ee04429d8afa651b33c", "hf_filter": null, "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:jurisprudence", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "senti_sk.prompt3_sentiment_3_class_sk|0": { "name": "senti_sk.prompt3_sentiment_3_class_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/senti-sk", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d", "hf_filter": null, "hf_avail_splits": [ "train", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:electrical_engineering", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt3_qa_sk:question-answering|0": { "name": "sklep.prompt3_qa_sk:question-answering", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "question-answering", "metrics": [ { "metric_name": "exact_match_slk_prefix", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "f1_slk", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 126, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:professional_psychology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:global_facts|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:global_facts", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:computer_security|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:computer_security", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:computer_security|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:computer_security", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:college_physics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:college_physics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "globalmmlu-lite.prompt2_mqa_single_correct_sk|0": { "name": "globalmmlu-lite.prompt2_mqa_single_correct_sk", "prompt_function": "prompt_fn", "hf_repo": "CohereLabs/Global-MMLU-Lite", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "600af1bff2d447c7fc3aee22f11f607560d5fcb7", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": "1" }, "mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_world_history", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:anatomy|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:anatomy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:security_studies|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:security_studies", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "movie_review.prompt3_sentiment_binary_sk|0": { "name": "movie_review.prompt3_sentiment_binary_sk", "prompt_function": "prompt_fn", "hf_repo": "mteb/slovak_movie_review_sentiment", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "29a7405aabcfd4860a51ae6f65a5650d63108f26", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:college_chemistry", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:astronomy|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:astronomy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:medical_genetics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:abstract_algebra", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:electrical_engineering", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_world_history", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:philosophy|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:philosophy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:human_sexuality", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:abstract_algebra", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:world_religions|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:world_religions", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "toxic_sk.prompt1_toxicity_sk|0": { "name": "toxic_sk.prompt1_toxicity_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/toxic-sk", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "2fb416e18785c128e7ec934c07a14c803a88cdb3", "hf_filter": null, "hf_avail_splits": [ "train", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "senti_sk.prompt1_sentiment_3_class_sk|0": { "name": "senti_sk.prompt1_sentiment_3_class_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/senti-sk", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d", "hf_filter": null, "hf_avail_splits": [ "train", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:college_biology|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:college_biology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:public_relations|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:public_relations", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:formal_logic|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:formal_logic", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_geography", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:prehistory|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:prehistory", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:econometrics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:econometrics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "arc.prompt1_mqa_single_correct_sk|0": { "name": "arc.prompt1_mqa_single_correct_sk", "prompt_function": "prompt_fn", "hf_repo": "jon-tow/okapi_arc_challenge", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": [ "solve" ], "scorer": "score", "sample_fields": "record_to_sample", "sample_to_fewshot": null, "filter": null, "hf_revision": "56aa31aedeb55f0635cc7c3c13f7f86b276f9a4e", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:business_ethics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:business_ethics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:jurisprudence", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "truthfulqa.prompt2_mqa_single_correct_sk:mc1|0": { "name": "truthfulqa.prompt2_mqa_single_correct_sk:mc1", "prompt_function": "prompt_fn", "hf_repo": "alexandrainst/m_truthfulqa", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "f0445d470f1925882b990f5f247fdcf288972f60", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "val" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:miscellaneous", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:astronomy|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:astronomy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_european_history", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "senti_sk.prompt2_sentiment_3_class_sk|0": { "name": "senti_sk.prompt2_sentiment_3_class_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/senti-sk", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d", "hf_filter": null, "hf_avail_splits": [ "train", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_european_history", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:formal_logic|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:formal_logic", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0": { "name": "sklep.prompt2_sentiment_binary_sk:sentiment-analysis", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "sentiment-analysis", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0": { "name": "sklep.prompt1_sentiment_binary_sk:sentiment-analysis", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "sentiment-analysis", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "skj1.prompt2_skj1_sk|0": { "name": "skj1.prompt2_skj1_sk", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/SKJ1", "hf_subset": "default", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "e615c60b415389b629703f82d3a79810dbde6734", "hf_filter": null, "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:international_law|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:international_law", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt1_nli_binary_sk:rte|0": { "name": "sklep.prompt1_nli_binary_sk:rte", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "rte", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:human_aging|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:human_aging", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:management|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:management", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:professional_accounting", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "truthfulqa.prompt1_mqa_single_correct_sk:mc1|0": { "name": "truthfulqa.prompt1_mqa_single_correct_sk:mc1", "prompt_function": "prompt_fn", "hf_repo": "alexandrainst/m_truthfulqa", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "f0445d470f1925882b990f5f247fdcf288972f60", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "val" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "senti_sk.prompt1_sentiment_binary_sk|0": { "name": "senti_sk.prompt1_sentiment_binary_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/senti-sk", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d", "hf_filter": "", "hf_avail_splits": [ "train", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:astronomy|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:astronomy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:human_aging|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:human_aging", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:formal_logic|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:formal_logic", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:sociology|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:sociology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:econometrics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:econometrics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:college_mathematics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:professional_psychology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:college_mathematics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "globalmmlu-lite.prompt3_mqa_single_correct_sk|0": { "name": "globalmmlu-lite.prompt3_mqa_single_correct_sk", "prompt_function": "prompt_fn", "hf_repo": "CohereLabs/Global-MMLU-Lite", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "600af1bff2d447c7fc3aee22f11f607560d5fcb7", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": "1" }, "mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:human_sexuality", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:computer_security|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:computer_security", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "toxic_sk.prompt2_toxicity_sk|0": { "name": "toxic_sk.prompt2_toxicity_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/toxic-sk", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "2fb416e18785c128e7ec934c07a14c803a88cdb3", "hf_filter": null, "hf_avail_splits": [ "train", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "multi_wiki_qa.prompt3_qa_sk|0": { "name": "multi_wiki_qa.prompt3_qa_sk", "prompt_function": "prompt_fn", "hf_repo": "alexandrainst/multi-wiki-qa", "hf_subset": "sk", "metrics": [ { "metric_name": "exact_match_slk_prefix", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "f1_slk", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "9bdb3350487e399bd1cd873e1395290d636db5ce", "hf_filter": "", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 126, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "financial_exams.prompt1_completion_sk|0": { "name": "financial_exams.prompt1_completion_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/slovak-financial-exam", "hf_subset": "default", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "7cdbfe635c3ed0d8b4df2ee04429d8afa651b33c", "hf_filter": null, "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:philosophy|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:philosophy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:moral_disputes", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:marketing|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:marketing", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "globalmmlu-lite.prompt1_mqa_single_correct_sk|0": { "name": "globalmmlu-lite.prompt1_mqa_single_correct_sk", "prompt_function": "prompt_fn", "hf_repo": "CohereLabs/Global-MMLU-Lite", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "600af1bff2d447c7fc3aee22f11f607560d5fcb7", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": "1" }, "mmlu.prompt1_mqa_single_correct_sk:professional_law|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:professional_law", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:machine_learning|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:machine_learning", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_physics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:management|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:management", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:sociology|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:sociology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "skj1.prompt3_skj1_sk|0": { "name": "skj1.prompt3_skj1_sk", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/SKJ1", "hf_subset": "default", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "e615c60b415389b629703f82d3a79810dbde6734", "hf_filter": null, "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "movie_review.prompt1_sentiment_binary_sk|0": { "name": "movie_review.prompt1_sentiment_binary_sk", "prompt_function": "prompt_fn", "hf_repo": "mteb/slovak_movie_review_sentiment", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "29a7405aabcfd4860a51ae6f65a5650d63108f26", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:anatomy|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:anatomy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:philosophy|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:philosophy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:human_aging|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:human_aging", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:professional_medicine", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:professional_psychology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:electrical_engineering", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:logical_fallacies", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt1_hate_speech_sk:hate-speech|0": { "name": "sklep.prompt1_hate_speech_sk:hate-speech", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "hate-speech", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:miscellaneous", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:world_religions|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:world_religions", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_biology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_world_history", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:college_biology|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:college_biology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:global_facts|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:global_facts", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:moral_disputes", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_geography", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:moral_scenarios", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:college_physics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:college_physics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:professional_medicine", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:logical_fallacies", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt3_nli_binary_sk:rte|0": { "name": "sklep.prompt3_nli_binary_sk:rte", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "rte", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "slovakcopa.prompt2_copa_sk|0": { "name": "slovakcopa.prompt2_copa_sk", "prompt_function": "prompt_fn", "hf_repo": "NaiveNeuron/SlovakCOPA", "hf_subset": "default", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "5c61e4cd9343f9b4fab623bdc50bb895cc170771", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "validation", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:jurisprudence", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "multi_wiki_qa.prompt1_qa_sk|0": { "name": "multi_wiki_qa.prompt1_qa_sk", "prompt_function": "prompt_fn", "hf_repo": "alexandrainst/multi-wiki-qa", "hf_subset": "sk", "metrics": [ { "metric_name": "exact_match_slk_prefix", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "f1_slk", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "9bdb3350487e399bd1cd873e1395290d636db5ce", "hf_filter": "", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 126, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_psychology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:global_facts|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:global_facts", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:sociology|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:sociology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_biology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:public_relations|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:public_relations", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_statistics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:anatomy|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:anatomy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:marketing|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:marketing", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:professional_accounting", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_geography", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "slovakcopa.prompt1_copa_sk|0": { "name": "slovakcopa.prompt1_copa_sk", "prompt_function": "prompt_fn", "hf_repo": "NaiveNeuron/SlovakCOPA", "hf_subset": "default", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "5c61e4cd9343f9b4fab623bdc50bb895cc170771", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "validation", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:nutrition|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:nutrition", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:machine_learning|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:machine_learning", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 } }, "summary_tasks": { "mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0": { "hashes": { "hash_examples": "81792091d0a281a1", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "197fc777a54fa451", "hash_cont_tokens": "2ca1254365bc4de8" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0": { "hashes": { "hash_examples": "ac8d3b0039dcb239", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "5db4f751889ea833", "hash_cont_tokens": "e63825b594496705" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "financial_exams.prompt2_completion_sk|0": { "hashes": { "hash_examples": "298560bc2b080ed4", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "0342f86b74762ed2", "hash_cont_tokens": "b8fcb2be5fc35e79" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:machine_learning|0": { "hashes": { "hash_examples": "9c6bb044db076a06", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "9d92fa8ac1790956", "hash_cont_tokens": "13ea8227d1f89b7a" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "belebele.prompt3_mqa_single_correct_sk|0": { "hashes": { "hash_examples": "40ce6eca2db6a52f", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "6461c4c38cc687cc", "hash_cont_tokens": "117c4c7e7c956a19" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0": { "hashes": { "hash_examples": "f7fbee8f9cf664d7", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "5f16ff9acfd0b051", "hash_cont_tokens": "dc758eff6372a6bd" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:professional_law|0": { "hashes": { "hash_examples": "645c5fc320444ab9", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "4d8fda65be8ecb47", "hash_cont_tokens": "680d5153c8206395" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0": { "hashes": { "hash_examples": "f2818986ee796bdd", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "c117c2660e917e7b", "hash_cont_tokens": "f070535f9aa37264" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0": { "hashes": { "hash_examples": "ce1327e99680eaa4", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "64ce9a865c4933e8", "hash_cont_tokens": "b0cad34573a530c2" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0": { "hashes": { "hash_examples": "648884be05478291", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "c306237952459a8c", "hash_cont_tokens": "0d3b3fe8f6a6e93e" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0": { "hashes": { "hash_examples": "ea7b1315dcf37a69", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "af88757d442d46fb", "hash_cont_tokens": "2a0c5d184ed08ac7" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0": { "hashes": { "hash_examples": "648884be05478291", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "dc75ba147ed27b16", "hash_cont_tokens": "a112819f21ae3613" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:professional_law|0": { "hashes": { "hash_examples": "645c5fc320444ab9", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "2ad645d70202f61c", "hash_cont_tokens": "568fc39fd238e02c" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "sklep.prompt1_qa_sk:question-answering|0": { "hashes": { "hash_examples": "8026b7009fd8d93f", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "9b2ecc66440c6655", "hash_cont_tokens": "7b155235596271e5" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0": { "hashes": { "hash_examples": "fc35facc4060af81", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "5019761d5bacf09d", "hash_cont_tokens": "22c11b68f032f39c" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0": { "hashes": { "hash_examples": "2e0d5171d0d40ca2", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "ed2271f4780c6c74", "hash_cont_tokens": "bdc0ee0b2faaa2e8" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0": { "hashes": { "hash_examples": "5bc6e3e3c9ba587f", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "3042c9a7af42d006", "hash_cont_tokens": "a86bd0a20a7d22a5" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0": { "hashes": { "hash_examples": "992e1937e8f9557c", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "0210a9fbd544fe47", "hash_cont_tokens": "e552bd0023154eb6" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0": { "hashes": { "hash_examples": "00f9ec3430780b27", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "7c85e86631c58d50", "hash_cont_tokens": "cd04bbd91a899720" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0": { "hashes": { "hash_examples": "ea73340f4d960eee", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "8973856ba2b7c3e4", "hash_cont_tokens": "73211ae3847f43b5" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "slovakcopa.prompt3_copa_sk|0": { "hashes": { "hash_examples": "19e8e1315421e50c", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "c14d1e258331eaae", "hash_cont_tokens": "4d37bc2a9714402e" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "movie_review.prompt2_sentiment_binary_sk|0": { "hashes": { "hash_examples": "f731da94d9560ab2", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "eda480540fb1d6c4", "hash_cont_tokens": "5ff999ac05d6ebee" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "sklep.prompt2_qa_sk:question-answering|0": { "hashes": { "hash_examples": "8026b7009fd8d93f", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "805c2f8789ee2559", "hash_cont_tokens": "b6a58ddf979e8542" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:college_biology|0": { "hashes": { "hash_examples": "8bd34873c0ac9972", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "ba08fe18dcb39b17", "hash_cont_tokens": "1c0532ee760fc3d2" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:nutrition|0": { "hashes": { "hash_examples": "47563eff5f4089bf", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "e5398ee043cbef07", "hash_cont_tokens": "0392303738faba09" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0": { "hashes": { "hash_examples": "27b6684dc6c0d1b3", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "8d522cc6cead753a", "hash_cont_tokens": "0ff90fe9e9aefbc0" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "arc.prompt3_mqa_single_correct_sk|0": { "hashes": { "hash_examples": "6725e9266ede5f20", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "b9a63364b1f284dd", "hash_cont_tokens": "15b124076d60386a" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:econometrics|0": { "hashes": { "hash_examples": "7d90b1f77d986bbe", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "db859ef4b587dde8", "hash_cont_tokens": "e774cef893436bb0" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:prehistory|0": { "hashes": { "hash_examples": "375708e013c71363", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "108f91cedf2751e3", "hash_cont_tokens": "e300daa822554479" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "toxic_sk.prompt3_toxicity_sk|0": { "hashes": { "hash_examples": "d01cdad28edb6cc2", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "b572324ccee144a3", "hash_cont_tokens": "894f32af192a6aba" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "belebele.prompt1_mqa_single_correct_sk|0": { "hashes": { "hash_examples": "40ce6eca2db6a52f", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "568128be863811c3", "hash_cont_tokens": "926371267f4e5206" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "arc.prompt2_mqa_single_correct_sk|0": { "hashes": { "hash_examples": "6725e9266ede5f20", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "7b3aca5e97b189fd", "hash_cont_tokens": "fecd44e229b2e8b5" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "multi_wiki_qa.prompt2_qa_sk|0": { "hashes": { "hash_examples": "f166d5d501f82d9e", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "5e1bd4b50a6cd47e", "hash_cont_tokens": "60f3560556af052d" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:management|0": { "hashes": { "hash_examples": "6be6db70af55c301", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "222ff13fe0318e8d", "hash_cont_tokens": "7a5047bcd8b04f08" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0": { "hashes": { "hash_examples": "1b29eb003b1f4bae", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "9ee3fdc1e5b7dafb", "hash_cont_tokens": "5ade4ebf51fa62e1" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0": { "hashes": { "hash_examples": "fc35facc4060af81", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "39550d80d1a77e43", "hash_cont_tokens": "5b1778c721f6f013" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "sklep.prompt3_nli_3_classes_sk:nli|0": { "hashes": { "hash_examples": "4ba8c3712f2beb9b", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "93b6e471405ec4a2", "hash_cont_tokens": "b00e20d4002c0576" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:college_medicine|0": { "hashes": { "hash_examples": "bfbac8bb6ff2adc7", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "90abb1fe093d34b9", "hash_cont_tokens": "86ebffd8f5e2f04a" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:business_ethics|0": { "hashes": { "hash_examples": "f891ffad3913452c", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "313df26edb3e563a", "hash_cont_tokens": "50713a1395a44b0a" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0": { "hashes": { "hash_examples": "f2818986ee796bdd", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "be7503adf6970be2", "hash_cont_tokens": "122abbf33ae2f3ff" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:international_law|0": { "hashes": { "hash_examples": "a3aec6cd58af642c", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "bf45b63fa78edc41", "hash_cont_tokens": "26ef649f0f6b3ee1" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:international_law|0": { "hashes": { "hash_examples": "a3aec6cd58af642c", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "ec8c1433297b3b71", "hash_cont_tokens": "0a6070b89f5afa81" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "truthfulqa.prompt3_mqa_single_correct_sk:mc1|0": { "hashes": { "hash_examples": "ee751399d6f4ae5f", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "cd647a82dbdd5bc7", "hash_cont_tokens": "69cd33952c06db63" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:prehistory|0": { "hashes": { "hash_examples": "375708e013c71363", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "08f1d64053a47ae0", "hash_cont_tokens": "58f83cb29db5c299" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:nutrition|0": { "hashes": { "hash_examples": "47563eff5f4089bf", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "c33f74fed807aefd", "hash_cont_tokens": "fd939458a7f74016" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0": { "hashes": { "hash_examples": "ea7b1315dcf37a69", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "3c3de4494a17e944", "hash_cont_tokens": "5eb735aace25626b" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0": { "hashes": { "hash_examples": "7af87a7ae1cb0e53", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "1246824958e77888", "hash_cont_tokens": "ecb050b141027f48" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0": { "hashes": { "hash_examples": "f2818986ee796bdd", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "83f120a531fb3e2a", "hash_cont_tokens": "774146b60df6da60" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:college_medicine|0": { "hashes": { "hash_examples": "bfbac8bb6ff2adc7", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "3b4c1c73f59b5469", "hash_cont_tokens": "6b97ef9c9d2810f6" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:college_medicine|0": { "hashes": { "hash_examples": "bfbac8bb6ff2adc7", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "1189b08a42e2f634", "hash_cont_tokens": "633598661bd73f11" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0": { "hashes": { "hash_examples": "7c4135abff305666", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "50d3dc7196dfff51", "hash_cont_tokens": "316ce90d2f2d0e0f" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "sklep.prompt2_nli_binary_sk:rte|0": { "hashes": { "hash_examples": "8ea06a4d7027e078", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "c0e09fc5609f1af2", "hash_cont_tokens": "8279eee4a64f6547" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0": { "hashes": { "hash_examples": "2e0d5171d0d40ca2", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "057a4329cdd6e58d", "hash_cont_tokens": "46f0e24c2c587524" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "sklep.prompt2_hate_speech_sk:hate-speech|0": { "hashes": { "hash_examples": "b3cd4f1ab55c772f", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "7505cc51168e73f6", "hash_cont_tokens": "8728167288d94f99" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0": { "hashes": { "hash_examples": "7f3582a44ea04a78", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "702dd3167af66b42", "hash_cont_tokens": "2600d71645cb0a4f" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0": { "hashes": { "hash_examples": "7c4135abff305666", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "a37f1fee9ddb74c7", "hash_cont_tokens": "6c4429127f00cd1d" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0": { "hashes": { "hash_examples": "648884be05478291", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "cad3680d064b5933", "hash_cont_tokens": "7365f46c693425df" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:marketing|0": { "hashes": { "hash_examples": "c4bcef0587679739", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "7e6350cc032822d9", "hash_cont_tokens": "415bf899898a2a8c" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0": { "hashes": { "hash_examples": "8d92d327d53b2933", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "248010b481a96ce3", "hash_cont_tokens": "5e3aaed86141a76b" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0": { "hashes": { "hash_examples": "7af87a7ae1cb0e53", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "d412e82c777fb52f", "hash_cont_tokens": "cf2aaf8aff7b14cc" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:virology|0": { "hashes": { "hash_examples": "7c5d14ef48d0e860", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "a9e052f5318d6b5e", "hash_cont_tokens": "3d5faee2fb339706" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0": { "hashes": { "hash_examples": "44459a57c28f5620", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "28139582f9049b37", "hash_cont_tokens": "1d9de2bbe333fc6a" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0": { "hashes": { "hash_examples": "f6a1eb6fc83b4ee7", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "c7778a40d8d53049", "hash_cont_tokens": "99db1895550f14b4" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0": { "hashes": { "hash_examples": "43cdbac391d8a1a9", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "5c800146e8f560f9", "hash_cont_tokens": "0a3cf0d638dc970f" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:world_religions|0": { "hashes": { "hash_examples": "c1bbf26f068bb598", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "61a75455c51f4ae7", "hash_cont_tokens": "63f3433623e54798" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "senti_sk.prompt3_sentiment_binary_sk|0": { "hashes": { "hash_examples": "33e538782d5b0e84", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "13c20804bf62b2ed", "hash_cont_tokens": "4f050f3d04b35265" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0": { "hashes": { "hash_examples": "992e1937e8f9557c", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "ea4a28c1ab9215f5", "hash_cont_tokens": "22ae6613b401fc8f" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "sklep.prompt1_nli_3_classes_sk:nli|0": { "hashes": { "hash_examples": "4ba8c3712f2beb9b", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "f7ab195fbc9b5768", "hash_cont_tokens": "fa7cc85c20cc1a10" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0": { "hashes": { "hash_examples": "1cb2a8b29830cfad", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "b98540648f767a25", "hash_cont_tokens": "99f2f20435299486" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0": { "hashes": { "hash_examples": "f6a1eb6fc83b4ee7", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "8c033a4d5267d677", "hash_cont_tokens": "5a8db146b973ff63" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0": { "hashes": { "hash_examples": "ea7b1315dcf37a69", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "25a0f18af8b6f2a0", "hash_cont_tokens": "d51a34da6c1fbf8b" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0": { "hashes": { "hash_examples": "c5e5e17570e2bb8c", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "c5949296a80706d7", "hash_cont_tokens": "c0f6ff91bafd7713" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:business_ethics|0": { "hashes": { "hash_examples": "f891ffad3913452c", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "05003e5e7e31aed0", "hash_cont_tokens": "1ece07f9dc1f9c22" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:public_relations|0": { "hashes": { "hash_examples": "74192a1a6507cb74", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "aeca9e8ab837f186", "hash_cont_tokens": "b5fed06ce9a0c457" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0": { "hashes": { "hash_examples": "00f9ec3430780b27", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "bc7b7df7d3928228", "hash_cont_tokens": "c418eef43e5ff47f" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "senti_sk.prompt2_sentiment_binary_sk|0": { "hashes": { "hash_examples": "48967df94707805e", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "0b0ee4739e0aa3ad", "hash_cont_tokens": "9c1b73a32b734550" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0": { "hashes": { "hash_examples": "44459a57c28f5620", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "bac621d5b19b49c9", "hash_cont_tokens": "f0de293fd5c86f70" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "belebele.prompt2_mqa_single_correct_sk|0": { "hashes": { "hash_examples": "40ce6eca2db6a52f", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "e1345cfe6c247089", "hash_cont_tokens": "84169ff79e214a23" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:virology|0": { "hashes": { "hash_examples": "7c5d14ef48d0e860", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "56615f3e5db47185", "hash_cont_tokens": "a946306d5cb5bd4b" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "sklep.prompt2_nli_3_classes_sk:nli|0": { "hashes": { "hash_examples": "4ba8c3712f2beb9b", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "23797da611879f2b", "hash_cont_tokens": "3d355a5136418900" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0": { "hashes": { "hash_examples": "3fc0b7fe5c48a36c", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "d4dfebb59911599c", "hash_cont_tokens": "be2b3240658584b3" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0": { "hashes": { "hash_examples": "1cb2a8b29830cfad", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "252942e119de6012", "hash_cont_tokens": "02fcb3d20e2d1fad" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0": { "hashes": { "hash_examples": "c50a05f1347547cb", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "7fc690f1285f2fe0", "hash_cont_tokens": "2da5682f64b00797" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0": { "hashes": { "hash_examples": "f66b2c8f1db1d216", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "29612ddcd200d9dd", "hash_cont_tokens": "cf0f2bbdb1fa7243" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0": { "hashes": { "hash_examples": "7af87a7ae1cb0e53", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "44c4c682673a0e5a", "hash_cont_tokens": "a03360803aaa6717" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0": { "hashes": { "hash_examples": "f84155f1bdc02fca", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "75941e07951d94b0", "hash_cont_tokens": "7f07bc5c0339f6bd" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0": { "hashes": { "hash_examples": "b19a89d99d848605", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "7dff559e3f16a8d9", "hash_cont_tokens": "8a0ef54196a324dc" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:security_studies|0": { "hashes": { "hash_examples": "9d7ec8fc0001ca04", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "9451192bda900677", "hash_cont_tokens": "6abf5a54c7a1ba7e" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "skj1.prompt1_skj1_sk|0": { "hashes": { "hash_examples": "b288f09270a11fde", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "bf070bfbf9410180", "hash_cont_tokens": "58372b1d0c345e75" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:college_physics|0": { "hashes": { "hash_examples": "c39cb2557a9b3972", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "16560bd9eded68a4", "hash_cont_tokens": "d24d3889990f62cb" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "sklep.prompt3_hate_speech_sk:hate-speech|0": { "hashes": { "hash_examples": "b3cd4f1ab55c772f", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "64454b3f71edc8f0", "hash_cont_tokens": "22e05c7d79fbf2b1" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0": { "hashes": { "hash_examples": "c50a05f1347547cb", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "a36778b8a176251b", "hash_cont_tokens": "83688fdf12bf6e6a" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:virology|0": { "hashes": { "hash_examples": "7c5d14ef48d0e860", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "88f2fe5f90545f4c", "hash_cont_tokens": "48606f764429804b" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0": { "hashes": { "hash_examples": "c50a05f1347547cb", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "ea6f8141161eff9c", "hash_cont_tokens": "03a7cf0dde79ab06" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:security_studies|0": { "hashes": { "hash_examples": "9d7ec8fc0001ca04", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "c00402398e7df06b", "hash_cont_tokens": "e66ce506f9da2a91" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "financial_exams.prompt3_completion_sk|0": { "hashes": { "hash_examples": "298560bc2b080ed4", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "9ccbde761f2b4c99", "hash_cont_tokens": "afb6d8f336cee813" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0": { "hashes": { "hash_examples": "a3ec8c0cb9af6b38", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "65b266e8828d2912", "hash_cont_tokens": "dc51ef05c787085a" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "senti_sk.prompt3_sentiment_3_class_sk|0": { "hashes": { "hash_examples": "77be771c44112b50", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "f774bb102b4544cf", "hash_cont_tokens": "09d400de44c1ae08" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0": { "hashes": { "hash_examples": "dc8d9d90c47f46e4", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "0af57f4163626315", "hash_cont_tokens": "2bb2bb33f8d2ea57" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "sklep.prompt3_qa_sk:question-answering|0": { "hashes": { "hash_examples": "387d6087cfc7dd8f", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "b267f5f74fcd520b", "hash_cont_tokens": "bb89dad0f13100d5" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0": { "hashes": { "hash_examples": "5ca55eb406065df7", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "5702376dbd94a73e", "hash_cont_tokens": "c397ae166819e71f" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:global_facts|0": { "hashes": { "hash_examples": "b92b2bc6891355a1", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "6f477d42372eb6a2", "hash_cont_tokens": "84414ca9c05007ff" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:computer_security|0": { "hashes": { "hash_examples": "190389c0bf2799c1", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "64a28c1d4ccfc4c8", "hash_cont_tokens": "e87564e25b1aad81" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:computer_security|0": { "hashes": { "hash_examples": "190389c0bf2799c1", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "9f6bc51ae575825f", "hash_cont_tokens": "0c1f857e6db1e56e" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:college_physics|0": { "hashes": { "hash_examples": "c39cb2557a9b3972", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "a9331ecdd1e4830e", "hash_cont_tokens": "3214166aa5eb6d7f" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "globalmmlu-lite.prompt2_mqa_single_correct_sk|0": { "hashes": { "hash_examples": "40308c6801250223", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "24091d8752abbeaa", "hash_cont_tokens": "ffea3262e8a4d687" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0": { "hashes": { "hash_examples": "2c57bfc497c8083c", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "b211d0815455fdd5", "hash_cont_tokens": "1cb2bcd292fe4c01" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:anatomy|0": { "hashes": { "hash_examples": "48e520a5e6a35fd8", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "9b167160771c1d25", "hash_cont_tokens": "41051fc8916208dd" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:security_studies|0": { "hashes": { "hash_examples": "9d7ec8fc0001ca04", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "ac2cb98935cc711b", "hash_cont_tokens": "5fc5c4ff5d04489f" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "movie_review.prompt3_sentiment_binary_sk|0": { "hashes": { "hash_examples": "cce02bac83d785bd", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "4965d78840c8cfd9", "hash_cont_tokens": "95529fe2cc8512eb" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0": { "hashes": { "hash_examples": "fc35facc4060af81", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "b81bff7429195938", "hash_cont_tokens": "45c38e83d9571bb5" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:astronomy|0": { "hashes": { "hash_examples": "ee6f5cfefae516d1", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "f33195ba9a67a08f", "hash_cont_tokens": "64a5122b714b43a4" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0": { "hashes": { "hash_examples": "44459a57c28f5620", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "d60d3344ad76e89e", "hash_cont_tokens": "6f9c1b83814ac536" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0": { "hashes": { "hash_examples": "1b29eb003b1f4bae", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "85ec8d09d026c1b4", "hash_cont_tokens": "9284bccbf99b5c4f" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0": { "hashes": { "hash_examples": "dc8d9d90c47f46e4", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "9caa457ae591ab77", "hash_cont_tokens": "0b49a2fe92118089" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0": { "hashes": { "hash_examples": "2c57bfc497c8083c", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "e0a45bbe0d3a0937", "hash_cont_tokens": "0f606616138bab99" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:philosophy|0": { "hashes": { "hash_examples": "2265c96c25374e0d", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "12ae1a98f4417384", "hash_cont_tokens": "43fe13827cf53aad" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0": { "hashes": { "hash_examples": "7f3582a44ea04a78", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "ba4b875526c4cf46", "hash_cont_tokens": "a5811fbb8878fe23" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0": { "hashes": { "hash_examples": "1b29eb003b1f4bae", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "b19a194aa1d50e97", "hash_cont_tokens": "1a7a59b45b0e8be9" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:world_religions|0": { "hashes": { "hash_examples": "c1bbf26f068bb598", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "f37bc31a030baa9a", "hash_cont_tokens": "5b86a700b4becc79" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0": { "hashes": { "hash_examples": "f7fbee8f9cf664d7", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "bcab0cefe636430a", "hash_cont_tokens": "3c8424cf270f9f3e" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "toxic_sk.prompt1_toxicity_sk|0": { "hashes": { "hash_examples": "d01cdad28edb6cc2", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "f0a31ed385787863", "hash_cont_tokens": "cca90921d1085e3c" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "senti_sk.prompt1_sentiment_3_class_sk|0": { "hashes": { "hash_examples": "b190fde299643895", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "dd2cf0eb910c26af", "hash_cont_tokens": "2773fbdc395f658f" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:college_biology|0": { "hashes": { "hash_examples": "8bd34873c0ac9972", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "01dcdfda57a14c51", "hash_cont_tokens": "e698a72c294db6d5" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:public_relations|0": { "hashes": { "hash_examples": "74192a1a6507cb74", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "8a3e9186ecfb12ec", "hash_cont_tokens": "54008dff8e859852" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0": { "hashes": { "hash_examples": "f7fbee8f9cf664d7", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "713a7980939a0201", "hash_cont_tokens": "e3095155cb624465" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:formal_logic|0": { "hashes": { "hash_examples": "89e12f421bdc146a", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "1bde99bb98c7b811", "hash_cont_tokens": "b89e4c9f6d5feb16" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0": { "hashes": { "hash_examples": "e2f6028f88e56ded", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "75b930014b12d245", "hash_cont_tokens": "c1c8f4b92381d0c7" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:prehistory|0": { "hashes": { "hash_examples": "375708e013c71363", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "15fb08f445126658", "hash_cont_tokens": "e865dbeccfdc7c97" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:econometrics|0": { "hashes": { "hash_examples": "7d90b1f77d986bbe", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "ded5b55309a02fc7", "hash_cont_tokens": "3c6714f49aacb296" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "arc.prompt1_mqa_single_correct_sk|0": { "hashes": { "hash_examples": "6725e9266ede5f20", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "41b53dec6ce643a8", "hash_cont_tokens": "28fe7acec29b9fb9" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:business_ethics|0": { "hashes": { "hash_examples": "f891ffad3913452c", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "6dd09ff1c6d50bcc", "hash_cont_tokens": "9e46ec69a1eaf930" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0": { "hashes": { "hash_examples": "a3ec8c0cb9af6b38", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "d05b196a041fc49c", "hash_cont_tokens": "47a93376504fa4e3" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "truthfulqa.prompt2_mqa_single_correct_sk:mc1|0": { "hashes": { "hash_examples": "ee751399d6f4ae5f", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "347d8c24b9b0fa76", "hash_cont_tokens": "67eb7937d8d6ede4" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0": { "hashes": { "hash_examples": "81792091d0a281a1", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "1446b11a62794dd5", "hash_cont_tokens": "d1658cdc2bbd7c21" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:astronomy|0": { "hashes": { "hash_examples": "ee6f5cfefae516d1", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "688bd5d3ae4f3327", "hash_cont_tokens": "52f4493b7f987643" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0": { "hashes": { "hash_examples": "3fc0b7fe5c48a36c", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "f12179ec42cdbd8b", "hash_cont_tokens": "5db3a0c85c74416c" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "senti_sk.prompt2_sentiment_3_class_sk|0": { "hashes": { "hash_examples": "e15f449b9f949296", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "31b44dbb87a1e410", "hash_cont_tokens": "ba0197d7d9005547" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0": { "hashes": { "hash_examples": "3fc0b7fe5c48a36c", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "680e80f0187d40b8", "hash_cont_tokens": "4d1786164b96bed1" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:formal_logic|0": { "hashes": { "hash_examples": "89e12f421bdc146a", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "2647ed914277ad4a", "hash_cont_tokens": "440826ea5784b5cd" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0": { "hashes": { "hash_examples": "c5e5e17570e2bb8c", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "514875b657d2a2e7", "hash_cont_tokens": "38f535b0d5304e6b" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0": { "hashes": { "hash_examples": "4f05c737d92acb1b", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "3d293b6cc7ba3944", "hash_cont_tokens": "a5674a7e851f8650" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0": { "hashes": { "hash_examples": "448c6360a63e82fc", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "52bdfcdaefdae0a8", "hash_cont_tokens": "d5bb4872feed277f" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "skj1.prompt2_skj1_sk|0": { "hashes": { "hash_examples": "b288f09270a11fde", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "3be1728f5c52d964", "hash_cont_tokens": "721ed1cd853c6e69" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:international_law|0": { "hashes": { "hash_examples": "a3aec6cd58af642c", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "70f3343aa62cfcc7", "hash_cont_tokens": "2dd13507c763ffa5" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "sklep.prompt1_nli_binary_sk:rte|0": { "hashes": { "hash_examples": "8ea06a4d7027e078", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "e0e9316ffd581271", "hash_cont_tokens": "be0550ba11635e11" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:human_aging|0": { "hashes": { "hash_examples": "1d0dd017926beb00", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "4f05f6e7a550c21a", "hash_cont_tokens": "54458199d9af0a5e" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:management|0": { "hashes": { "hash_examples": "6be6db70af55c301", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "62299f6e7e6dd7ad", "hash_cont_tokens": "0b996d5a7366c705" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0": { "hashes": { "hash_examples": "ce1327e99680eaa4", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "262eefd819afdf80", "hash_cont_tokens": "6da4120628b9ad37" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "truthfulqa.prompt1_mqa_single_correct_sk:mc1|0": { "hashes": { "hash_examples": "ee751399d6f4ae5f", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "8fb4041d9b70121f", "hash_cont_tokens": "caabb41ccba0a6ac" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "senti_sk.prompt1_sentiment_binary_sk|0": { "hashes": { "hash_examples": "394f7fff1d43e549", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "d3a66d1e658ad28e", "hash_cont_tokens": "7e88c9420eb5058d" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:astronomy|0": { "hashes": { "hash_examples": "ee6f5cfefae516d1", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "d0ad75a60f59cdf4", "hash_cont_tokens": "6986841198743675" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:human_aging|0": { "hashes": { "hash_examples": "1d0dd017926beb00", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "a93f9a4c95f94c4e", "hash_cont_tokens": "fab496ef83c3edef" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:formal_logic|0": { "hashes": { "hash_examples": "89e12f421bdc146a", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "ffe03d00391d95e2", "hash_cont_tokens": "b935ca66658fcebb" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:sociology|0": { "hashes": { "hash_examples": "dff8fca926f7b793", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "b2f3eaea6d328989", "hash_cont_tokens": "8136d16dda96abd7" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:econometrics|0": { "hashes": { "hash_examples": "7d90b1f77d986bbe", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "df280927b76e86c3", "hash_cont_tokens": "d7f0770f9a0acc43" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0": { "hashes": { "hash_examples": "ac8d3b0039dcb239", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "348619f08a3cb49f", "hash_cont_tokens": "2587b22c786990ab" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0": { "hashes": { "hash_examples": "5bc6e3e3c9ba587f", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "de95e8a190e7106e", "hash_cont_tokens": "ab9a76e59c0de440" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0": { "hashes": { "hash_examples": "5ca55eb406065df7", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "af10b44637f3d938", "hash_cont_tokens": "ba9094d4854ad209" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0": { "hashes": { "hash_examples": "5bc6e3e3c9ba587f", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "3f9133a48bea2708", "hash_cont_tokens": "99d4ea17fc3b67b1" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "globalmmlu-lite.prompt3_mqa_single_correct_sk|0": { "hashes": { "hash_examples": "40308c6801250223", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "f048bcc8c8407fab", "hash_cont_tokens": "5eb598a9cf4b1a43" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0": { "hashes": { "hash_examples": "1cb2a8b29830cfad", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "cc534b0b484b0889", "hash_cont_tokens": "895b0ada8425f24e" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0": { "hashes": { "hash_examples": "2e0d5171d0d40ca2", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "4c0430a1d48a772f", "hash_cont_tokens": "118cc18af5afd220" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0": { "hashes": { "hash_examples": "7f3582a44ea04a78", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "a3caec54057832f7", "hash_cont_tokens": "7a2ebf934c6ddd47" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:computer_security|0": { "hashes": { "hash_examples": "190389c0bf2799c1", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "29e630b4667d806e", "hash_cont_tokens": "fc8bb5f88f776ddd" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "toxic_sk.prompt2_toxicity_sk|0": { "hashes": { "hash_examples": "d01cdad28edb6cc2", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "7f740765062a487f", "hash_cont_tokens": "bace2bcdcd43efff" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "multi_wiki_qa.prompt3_qa_sk|0": { "hashes": { "hash_examples": "ee9acf278d86ea6b", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "a691374d2eecdcc1", "hash_cont_tokens": "613fd2b1ac9a5cfd" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "financial_exams.prompt1_completion_sk|0": { "hashes": { "hash_examples": "298560bc2b080ed4", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "2b48a01eefc2cb5a", "hash_cont_tokens": "5e32bbcae590cd3d" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:philosophy|0": { "hashes": { "hash_examples": "2265c96c25374e0d", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "c529c877230b655e", "hash_cont_tokens": "224aeb6b81f7af4a" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0": { "hashes": { "hash_examples": "b19a89d99d848605", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "9cca5de891f88280", "hash_cont_tokens": "46863e3a251ea603" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:marketing|0": { "hashes": { "hash_examples": "c4bcef0587679739", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "a61868461bf00ce3", "hash_cont_tokens": "a975feda43d08a31" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "globalmmlu-lite.prompt1_mqa_single_correct_sk|0": { "hashes": { "hash_examples": "40308c6801250223", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "904359bdb1a2a0c5", "hash_cont_tokens": "91569cbd7a98e2e7" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:professional_law|0": { "hashes": { "hash_examples": "645c5fc320444ab9", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "fecb378080561a4e", "hash_cont_tokens": "7efe7abd05372091" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:machine_learning|0": { "hashes": { "hash_examples": "9c6bb044db076a06", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "695fbc65bb4b4749", "hash_cont_tokens": "1773964b488ee162" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0": { "hashes": { "hash_examples": "43cdbac391d8a1a9", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "a53ed6b9166977dd", "hash_cont_tokens": "811e17a82722f4b1" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0": { "hashes": { "hash_examples": "c5e5e17570e2bb8c", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "7e294dff86f1a2d3", "hash_cont_tokens": "d5c498abba098875" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0": { "hashes": { "hash_examples": "992e1937e8f9557c", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "205358a7cc994b13", "hash_cont_tokens": "0e06e5533f04bd97" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:management|0": { "hashes": { "hash_examples": "6be6db70af55c301", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "c254897027f8e08a", "hash_cont_tokens": "e957e86212786d95" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:sociology|0": { "hashes": { "hash_examples": "dff8fca926f7b793", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "db5ae2473d1f67fc", "hash_cont_tokens": "4195ee7cc86517a2" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0": { "hashes": { "hash_examples": "43cdbac391d8a1a9", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "cfcede6c298395d7", "hash_cont_tokens": "256e6972f4cf49d5" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "skj1.prompt3_skj1_sk|0": { "hashes": { "hash_examples": "b288f09270a11fde", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "8e25eaf65ffc8542", "hash_cont_tokens": "fdaf8c910880d507" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "movie_review.prompt1_sentiment_binary_sk|0": { "hashes": { "hash_examples": "ec65ab0c1f9d54f6", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "6f783ecc2d68c0d4", "hash_cont_tokens": "eaac3ce2d9682b3c" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:anatomy|0": { "hashes": { "hash_examples": "48e520a5e6a35fd8", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "df796a450e66859d", "hash_cont_tokens": "70ee668bfc0bf1a1" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0": { "hashes": { "hash_examples": "f84155f1bdc02fca", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "7aabc5016dc3937e", "hash_cont_tokens": "52cde7c1b5f52f44" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:philosophy|0": { "hashes": { "hash_examples": "2265c96c25374e0d", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "7dad55635ca272a2", "hash_cont_tokens": "a0978611d159be0b" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:human_aging|0": { "hashes": { "hash_examples": "1d0dd017926beb00", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "0cdb4ea308771c51", "hash_cont_tokens": "c3223f47a2379678" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0": { "hashes": { "hash_examples": "ac8d3b0039dcb239", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "09ef2052a952a1f4", "hash_cont_tokens": "efb1518f8c99d455" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0": { "hashes": { "hash_examples": "f84155f1bdc02fca", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "ec80ae616fd5c56b", "hash_cont_tokens": "1f31bbdf0bd4bc9f" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0": { "hashes": { "hash_examples": "ea73340f4d960eee", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "8a7945fdabc42eb4", "hash_cont_tokens": "07e1a1a849acff1c" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0": { "hashes": { "hash_examples": "5ca55eb406065df7", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "00c9648dbca90eb4", "hash_cont_tokens": "83b5352b627600c0" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0": { "hashes": { "hash_examples": "dc8d9d90c47f46e4", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "a83a255e12ee5c30", "hash_cont_tokens": "db8f53776e03132c" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0": { "hashes": { "hash_examples": "8d92d327d53b2933", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "9c06353f63194270", "hash_cont_tokens": "93f4268a9a43e51b" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "sklep.prompt1_hate_speech_sk:hate-speech|0": { "hashes": { "hash_examples": "75602e70ff2fabe7", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "57dde360cdf42787", "hash_cont_tokens": "d145a63fa10283aa" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0": { "hashes": { "hash_examples": "81792091d0a281a1", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "6d03015e20d48f39", "hash_cont_tokens": "f3c8d6a4f1f41790" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:world_religions|0": { "hashes": { "hash_examples": "c1bbf26f068bb598", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "ac5577442c9ac24e", "hash_cont_tokens": "d839a2024ab476f9" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0": { "hashes": { "hash_examples": "27b6684dc6c0d1b3", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "d88fa2f69f58c729", "hash_cont_tokens": "6057522afb931155" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0": { "hashes": { "hash_examples": "2c57bfc497c8083c", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "92997f9687899a24", "hash_cont_tokens": "213d63dc03cb0502" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:college_biology|0": { "hashes": { "hash_examples": "8bd34873c0ac9972", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "a8f972b53e98ef2a", "hash_cont_tokens": "64e41340c060fb2c" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:global_facts|0": { "hashes": { "hash_examples": "b92b2bc6891355a1", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "2c2346af6175d9a6", "hash_cont_tokens": "02b1ec2d4956227f" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0": { "hashes": { "hash_examples": "b19a89d99d848605", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "011c1542b14a0f48", "hash_cont_tokens": "0dd254aecec04233" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0": { "hashes": { "hash_examples": "e2f6028f88e56ded", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "0694ec2172ab5650", "hash_cont_tokens": "c660ab315ca526ef" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0": { "hashes": { "hash_examples": "f6a1eb6fc83b4ee7", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "bd2bf17bbe995b0a", "hash_cont_tokens": "89e953ec48e709f3" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:college_physics|0": { "hashes": { "hash_examples": "c39cb2557a9b3972", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "6e8c68e90ea666f3", "hash_cont_tokens": "77498a00a20ba9ef" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0": { "hashes": { "hash_examples": "ea73340f4d960eee", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "6959bb25755ad88e", "hash_cont_tokens": "0e6b7a05add605a8" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0": { "hashes": { "hash_examples": "8d92d327d53b2933", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "e18b35af08465d62", "hash_cont_tokens": "3754583dd7de6bb2" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "sklep.prompt3_nli_binary_sk:rte|0": { "hashes": { "hash_examples": "8ea06a4d7027e078", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "9b7b2b80c583cf75", "hash_cont_tokens": "67142a1c5de16574" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "slovakcopa.prompt2_copa_sk|0": { "hashes": { "hash_examples": "3adee78dd6203724", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "248b9d8a04f8c062", "hash_cont_tokens": "20ad2370f3961274" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0": { "hashes": { "hash_examples": "a3ec8c0cb9af6b38", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "fbfa3f6159a2611c", "hash_cont_tokens": "a521f941e5eee9df" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "multi_wiki_qa.prompt1_qa_sk|0": { "hashes": { "hash_examples": "f166d5d501f82d9e", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "a787e89b7b7e6958", "hash_cont_tokens": "f9d012e1e6a49349" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0": { "hashes": { "hash_examples": "7c4135abff305666", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "80e30ebae0872849", "hash_cont_tokens": "4f427c2cde109462" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:global_facts|0": { "hashes": { "hash_examples": "b92b2bc6891355a1", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "39c0d368486ca8f0", "hash_cont_tokens": "67f5a60662be2e7d" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:sociology|0": { "hashes": { "hash_examples": "dff8fca926f7b793", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "0966718588555e05", "hash_cont_tokens": "35435678522c4939" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0": { "hashes": { "hash_examples": "27b6684dc6c0d1b3", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "abd87e6ae42adf81", "hash_cont_tokens": "e3ef820b35709164" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt1_mqa_single_correct_sk:public_relations|0": { "hashes": { "hash_examples": "74192a1a6507cb74", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "2222531cc6e9cd92", "hash_cont_tokens": "c430b74fedea99c2" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0": { "hashes": { "hash_examples": "00f9ec3430780b27", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "fb4be485e3061b23", "hash_cont_tokens": "4d687025986f0143" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:anatomy|0": { "hashes": { "hash_examples": "48e520a5e6a35fd8", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "cee5c9ddd78fd571", "hash_cont_tokens": "02da6630b897eeb5" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:marketing|0": { "hashes": { "hash_examples": "c4bcef0587679739", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "2b8aa9ed12f6b266", "hash_cont_tokens": "e7a7c554c130b919" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0": { "hashes": { "hash_examples": "ce1327e99680eaa4", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "ee5e80f1825f089c", "hash_cont_tokens": "51ca5d6d2b8ca9ad" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0": { "hashes": { "hash_examples": "e2f6028f88e56ded", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "8283452d92579ef1", "hash_cont_tokens": "27454841e4594359" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "slovakcopa.prompt1_copa_sk|0": { "hashes": { "hash_examples": "37b56578071ef6b2", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "8c6a5ae3cc5489ef", "hash_cont_tokens": "47e39104d5b846c3" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt2_mqa_single_correct_sk:nutrition|0": { "hashes": { "hash_examples": "47563eff5f4089bf", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "751607ba52c4d615", "hash_cont_tokens": "e750e09146977427" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 }, "mmlu.prompt3_mqa_single_correct_sk:machine_learning|0": { "hashes": { "hash_examples": "9c6bb044db076a06", "hash_full_prompts": "ef46db3751d8e999", "hash_input_tokens": "8b56dca09a4cead4", "hash_cont_tokens": "a8155bb958c58d4c" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 } }, "summary_general": { "hashes": { "hash_examples": "316f54894dcb6f0d", "hash_full_prompts": "ba7f9b257c923f55", "hash_input_tokens": "8de2f010e9daf7fe", "hash_cont_tokens": "1f06ad3e4b80f7cc" }, "truncated": 0, "non_truncated": 0, "padded": 0, "non_padded": 0 } }