Document six completed LR runs as preparation for full Aya training on Perun. Record selection of LR 0.00010 by lowest final validation loss, selected-model lm-eval and partial SK-bench results, reproducible provenance, and dataset-access/context-truncation limitations. Preserve existing 30k and Titan material.
18023 lines
623 KiB
JSON
18023 lines
623 KiB
JSON
{
|
|
"config_general": {
|
|
"lighteval_sha": "?",
|
|
"num_fewshot_seeds": 1,
|
|
"max_samples": null,
|
|
"job_id": "0",
|
|
"start_time": 796668.90423246,
|
|
"end_time": 807588.555648873,
|
|
"total_evaluation_time_secondes": "10919.65141641302",
|
|
"model_config": {
|
|
"model_name": "/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010",
|
|
"generation_parameters": {
|
|
"num_blocks": null,
|
|
"block_size": null,
|
|
"early_stopping": null,
|
|
"repetition_penalty": null,
|
|
"frequency_penalty": null,
|
|
"length_penalty": null,
|
|
"presence_penalty": null,
|
|
"max_new_tokens": null,
|
|
"min_new_tokens": null,
|
|
"seed": null,
|
|
"stop_tokens": null,
|
|
"temperature": 0,
|
|
"top_k": null,
|
|
"min_p": null,
|
|
"top_p": null,
|
|
"truncate_prompt": null,
|
|
"cache_implementation": null,
|
|
"response_format": null,
|
|
"skip_special_tokens": true
|
|
},
|
|
"system_prompt": null,
|
|
"cache_dir": "~/.cache/huggingface/lighteval",
|
|
"tokenizer": null,
|
|
"revision": "main",
|
|
"dtype": "bfloat16",
|
|
"tensor_parallel_size": 1,
|
|
"data_parallel_size": 1,
|
|
"pipeline_parallel_size": 1,
|
|
"gpu_memory_utilization": 0.85,
|
|
"enable_prefix_caching": null,
|
|
"max_model_length": 32768,
|
|
"quantization": null,
|
|
"load_format": null,
|
|
"swap_space": 4,
|
|
"seed": 1234,
|
|
"trust_remote_code": false,
|
|
"add_special_tokens": true,
|
|
"multichoice_continuations_start_space": true,
|
|
"pairwise_tokenization": false,
|
|
"max_num_seqs": 128,
|
|
"max_num_batched_tokens": 2048,
|
|
"subfolder": null,
|
|
"is_async": false,
|
|
"override_chat_template": null,
|
|
"enforce_eager": true
|
|
},
|
|
"model_name": "/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010"
|
|
},
|
|
"results": {
|
|
"mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0": {
|
|
"em_relaxed": 0.17843388960205392,
|
|
"em_relaxed_stderr": 0.013726826073077934,
|
|
"em": 0.055198973042362,
|
|
"em_stderr": 0.00818739755268085,
|
|
"em_conservative": 0.17715019255455713,
|
|
"em_conservative_stderr": 0.013688041138952063
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0": {
|
|
"em_relaxed": 0.06166219839142091,
|
|
"em_relaxed_stderr": 0.012471469772330128,
|
|
"em": 0.005361930294906166,
|
|
"em_stderr": 0.003786357797685454,
|
|
"em_conservative": 0.058981233243967826,
|
|
"em_conservative_stderr": 0.012214750400069162
|
|
},
|
|
"financial_exams.prompt2_completion_sk|0": {
|
|
"em_relaxed": 0.2383808095952024,
|
|
"em_relaxed_stderr": 0.011670501255680532,
|
|
"em": 0.07571214392803598,
|
|
"em_stderr": 0.007245549384888845,
|
|
"em_conservative": 0.20539730134932535,
|
|
"em_conservative_stderr": 0.011065150105060274
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:machine_learning|0": {
|
|
"em_relaxed": 0.11607142857142858,
|
|
"em_relaxed_stderr": 0.03040253249894973,
|
|
"em": 0.008928571428571428,
|
|
"em_stderr": 0.008928571428571423,
|
|
"em_conservative": 0.10714285714285714,
|
|
"em_conservative_stderr": 0.02935696201881377
|
|
},
|
|
"belebele.prompt3_mqa_single_correct_sk|0": {
|
|
"em_relaxed": 0.21333333333333335,
|
|
"em_relaxed_stderr": 0.013662962863011163,
|
|
"em": 0.03,
|
|
"em_stderr": 0.005689402360648914,
|
|
"em_conservative": 0.2111111111111111,
|
|
"em_conservative_stderr": 0.013610798969328486
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0": {
|
|
"em_relaxed": 0.15444015444015444,
|
|
"em_relaxed_stderr": 0.022497923516297567,
|
|
"em": 0.011583011583011582,
|
|
"em_stderr": 0.006661484058899665,
|
|
"em_conservative": 0.15444015444015444,
|
|
"em_conservative_stderr": 0.022497923516297567
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_law|0": {
|
|
"em_relaxed": 0.1340734415029889,
|
|
"em_relaxed_stderr": 0.009961367847349073,
|
|
"em": 0.0017079419299743809,
|
|
"em_stderr": 0.0012071811002646103,
|
|
"em_conservative": 0.12809564474807855,
|
|
"em_conservative_stderr": 0.009770317447740377
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0": {
|
|
"em_relaxed": 0.15,
|
|
"em_relaxed_stderr": 0.035887028128263714,
|
|
"em": 0.01,
|
|
"em_stderr": 0.009999999999999998,
|
|
"em_conservative": 0.11,
|
|
"em_conservative_stderr": 0.031446603773522035
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0": {
|
|
"em_relaxed": 0.15217391304347827,
|
|
"em_relaxed_stderr": 0.02165993479655374,
|
|
"em": 0.0036231884057971015,
|
|
"em_stderr": 0.0036231884057970937,
|
|
"em_conservative": 0.14130434782608695,
|
|
"em_conservative_stderr": 0.02100540442245809
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0": {
|
|
"em_relaxed": 0.12280701754385964,
|
|
"em_relaxed_stderr": 0.04385964912280699,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.12280701754385964,
|
|
"em_conservative_stderr": 0.04385964912280699
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0": {
|
|
"em_relaxed": 0.08585858585858586,
|
|
"em_relaxed_stderr": 0.019960225563172885,
|
|
"em": 0.010101010101010102,
|
|
"em_stderr": 0.007124341521250813,
|
|
"em_conservative": 0.08080808080808081,
|
|
"em_conservative_stderr": 0.019417681889724533
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0": {
|
|
"em_relaxed": 0.21052631578947367,
|
|
"em_relaxed_stderr": 0.05447885994808057,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.21052631578947367,
|
|
"em_conservative_stderr": 0.05447885994808057
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_law|0": {
|
|
"em_relaxed": 0.14175918018787362,
|
|
"em_relaxed_stderr": 0.01019734819812428,
|
|
"em": 0.004269854824935952,
|
|
"em_stderr": 0.001906270167826036,
|
|
"em_conservative": 0.14175918018787362,
|
|
"em_conservative_stderr": 0.01019734819812428
|
|
},
|
|
"sklep.prompt1_qa_sk:question-answering|0": {
|
|
"exact_match_slk_prefix": 0.16456224564332672,
|
|
"exact_match_slk_prefix_stderr": 0.003787861300566259,
|
|
"f1_slk": 0.3730151221096053,
|
|
"f1_slk_stderr": 0.003855179221962441
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0": {
|
|
"em_relaxed": 0.14,
|
|
"em_relaxed_stderr": 0.03487350880197772,
|
|
"em": 0.09,
|
|
"em_stderr": 0.028762349126466143,
|
|
"em_conservative": 0.14,
|
|
"em_conservative_stderr": 0.03487350880197772
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0": {
|
|
"em_relaxed": 0.17511520737327188,
|
|
"em_relaxed_stderr": 0.025860189581375218,
|
|
"em": 0.02304147465437788,
|
|
"em_stderr": 0.010208603193741356,
|
|
"em_conservative": 0.17511520737327188,
|
|
"em_conservative_stderr": 0.025860189581375218
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0": {
|
|
"em_relaxed": 0.04040404040404041,
|
|
"em_relaxed_stderr": 0.019890402239668436,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.030303030303030304,
|
|
"em_conservative_stderr": 0.017316017316017295
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0": {
|
|
"em_relaxed": 0.14666666666666667,
|
|
"em_relaxed_stderr": 0.02898223889753507,
|
|
"em": 0.04,
|
|
"em_stderr": 0.016053601490175616,
|
|
"em_conservative": 0.14666666666666667,
|
|
"em_conservative_stderr": 0.02898223889753507
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0": {
|
|
"em_relaxed": 0.1822429906542056,
|
|
"em_relaxed_stderr": 0.026451344562470138,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.17289719626168223,
|
|
"em_conservative_stderr": 0.025910984835548666
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0": {
|
|
"em_relaxed": 0.19696969696969696,
|
|
"em_relaxed_stderr": 0.024523802005628457,
|
|
"em": 0.003787878787878788,
|
|
"em_stderr": 0.003787878787878786,
|
|
"em_conservative": 0.19696969696969696,
|
|
"em_conservative_stderr": 0.024523802005628457
|
|
},
|
|
"slovakcopa.prompt3_copa_sk|0": {
|
|
"em_relaxed": 0.086,
|
|
"em_relaxed_stderr": 0.01255081854271602,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.084,
|
|
"em_conservative_stderr": 0.01241758401564368
|
|
},
|
|
"movie_review.prompt2_sentiment_binary_sk|0": {
|
|
"em_norm_relaxed": 0.48306772908366535,
|
|
"em_norm_relaxed_stderr": 0.011154424009707972,
|
|
"em_norm": 0.01045816733067729,
|
|
"em_norm_stderr": 0.002270759742696006,
|
|
"em_conservative_norm": 0.47758964143426297,
|
|
"em_conservative_norm_stderr": 0.011149609394535145
|
|
},
|
|
"sklep.prompt2_qa_sk:question-answering|0": {
|
|
"exact_match_slk_prefix": 0.16518835437754356,
|
|
"exact_match_slk_prefix_stderr": 0.003793637935655721,
|
|
"f1_slk": 0.3751701402109962,
|
|
"f1_slk_stderr": 0.0038611217329234445
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_biology|0": {
|
|
"em_relaxed": 0.15492957746478872,
|
|
"em_relaxed_stderr": 0.030472217725828407,
|
|
"em": 0.028169014084507043,
|
|
"em_stderr": 0.013933866272175799,
|
|
"em_conservative": 0.15492957746478872,
|
|
"em_conservative_stderr": 0.030472217725828407
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:nutrition|0": {
|
|
"em_relaxed": 0.18811881188118812,
|
|
"em_relaxed_stderr": 0.022488416775417838,
|
|
"em": 0.036303630363036306,
|
|
"em_stderr": 0.010763206393659715,
|
|
"em_conservative": 0.18811881188118812,
|
|
"em_conservative_stderr": 0.022488416775417838
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0": {
|
|
"em_relaxed": 0.24822695035460993,
|
|
"em_relaxed_stderr": 0.025770015644290406,
|
|
"em": 0.04964539007092199,
|
|
"em_stderr": 0.012957736571882147,
|
|
"em_conservative": 0.24113475177304963,
|
|
"em_conservative_stderr": 0.02551873104953778
|
|
},
|
|
"arc.prompt3_mqa_single_correct_sk|0": {
|
|
"em_relaxed": 0.1770744225834046,
|
|
"em_relaxed_stderr": 0.011169578281264629,
|
|
"em": 0.022241231822070145,
|
|
"em_stderr": 0.004314931013145922,
|
|
"em_conservative": 0.17621899059024806,
|
|
"em_conservative_stderr": 0.011148355810710043
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:econometrics|0": {
|
|
"em_relaxed": 0.10714285714285714,
|
|
"em_relaxed_stderr": 0.02935696201881377,
|
|
"em": 0.017857142857142856,
|
|
"em_stderr": 0.012569900164490043,
|
|
"em_conservative": 0.09821428571428571,
|
|
"em_conservative_stderr": 0.02824733740077989
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:prehistory|0": {
|
|
"em_relaxed": 0.18808777429467086,
|
|
"em_relaxed_stderr": 0.021913980163123535,
|
|
"em": 0.06583072100313479,
|
|
"em_stderr": 0.013906362189890747,
|
|
"em_conservative": 0.18808777429467086,
|
|
"em_conservative_stderr": 0.021913980163123535
|
|
},
|
|
"toxic_sk.prompt3_toxicity_sk|0": {
|
|
"em_norm_relaxed": 0.4960272417707151,
|
|
"em_norm_relaxed_stderr": 0.016854464514836336,
|
|
"em_norm": 0.007945516458569807,
|
|
"em_norm_stderr": 0.00299286751118699,
|
|
"em_conservative_norm": 0.4960272417707151,
|
|
"em_conservative_norm_stderr": 0.016854464514836336
|
|
},
|
|
"belebele.prompt1_mqa_single_correct_sk|0": {
|
|
"em_relaxed": 0.21555555555555556,
|
|
"em_relaxed_stderr": 0.01371452783236968,
|
|
"em": 0.02,
|
|
"em_stderr": 0.004669261421755523,
|
|
"em_conservative": 0.21555555555555556,
|
|
"em_conservative_stderr": 0.01371452783236968
|
|
},
|
|
"arc.prompt2_mqa_single_correct_sk|0": {
|
|
"em_relaxed": 0.22412318220701455,
|
|
"em_relaxed_stderr": 0.012201644195165715,
|
|
"em": 0.03507271171941831,
|
|
"em_stderr": 0.005382828530594407,
|
|
"em_conservative": 0.223267750213858,
|
|
"em_conservative_stderr": 0.012185048029719049
|
|
},
|
|
"multi_wiki_qa.prompt2_qa_sk|0": {
|
|
"exact_match_slk_prefix": 0.36001600960576347,
|
|
"exact_match_slk_prefix_stderr": 0.006791008078277957,
|
|
"f1_slk": 0.5090494926903776,
|
|
"f1_slk_stderr": 0.005852156127470673
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:management|0": {
|
|
"em_relaxed": 0.15,
|
|
"em_relaxed_stderr": 0.035887028128263686,
|
|
"em": 0.04,
|
|
"em_stderr": 0.019694638556693237,
|
|
"em_conservative": 0.15,
|
|
"em_conservative_stderr": 0.035887028128263686
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0": {
|
|
"em_relaxed": 0.06,
|
|
"em_relaxed_stderr": 0.023868325657594173,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.06,
|
|
"em_conservative_stderr": 0.023868325657594173
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0": {
|
|
"em_relaxed": 0.08,
|
|
"em_relaxed_stderr": 0.027265992434429093,
|
|
"em": 0.04,
|
|
"em_stderr": 0.019694638556693227,
|
|
"em_conservative": 0.08,
|
|
"em_conservative_stderr": 0.027265992434429093
|
|
},
|
|
"sklep.prompt3_nli_3_classes_sk:nli|0": {
|
|
"em_norm_relaxed": 0.001398880895283773,
|
|
"em_norm_relaxed_stderr": 0.0005284101391342323,
|
|
"em_norm": 0.0,
|
|
"em_norm_stderr": 0.0,
|
|
"em_conservative_norm": 0.001398880895283773,
|
|
"em_conservative_norm_stderr": 0.0005284101391342323
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_medicine|0": {
|
|
"em_relaxed": 0.1511627906976744,
|
|
"em_relaxed_stderr": 0.027392810919920886,
|
|
"em": 0.011627906976744186,
|
|
"em_stderr": 0.008198095184581854,
|
|
"em_conservative": 0.1511627906976744,
|
|
"em_conservative_stderr": 0.027392810919920886
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:business_ethics|0": {
|
|
"em_relaxed": 0.22,
|
|
"em_relaxed_stderr": 0.0416333199893227,
|
|
"em": 0.03,
|
|
"em_stderr": 0.017144660799776522,
|
|
"em_conservative": 0.21,
|
|
"em_conservative_stderr": 0.04093601807403326
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0": {
|
|
"em_relaxed": 0.15,
|
|
"em_relaxed_stderr": 0.035887028128263714,
|
|
"em": 0.03,
|
|
"em_stderr": 0.01714466079977653,
|
|
"em_conservative": 0.11,
|
|
"em_conservative_stderr": 0.031446603773522035
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:international_law|0": {
|
|
"em_relaxed": 0.15254237288135594,
|
|
"em_relaxed_stderr": 0.03324002290477697,
|
|
"em": 0.0423728813559322,
|
|
"em_stderr": 0.018622984668462274,
|
|
"em_conservative": 0.15254237288135594,
|
|
"em_conservative_stderr": 0.03324002290477697
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:international_law|0": {
|
|
"em_relaxed": 0.13559322033898305,
|
|
"em_relaxed_stderr": 0.03165083260501198,
|
|
"em": 0.03389830508474576,
|
|
"em_stderr": 0.016730444637044904,
|
|
"em_conservative": 0.13559322033898305,
|
|
"em_conservative_stderr": 0.03165083260501198
|
|
},
|
|
"truthfulqa.prompt3_mqa_single_correct_sk:mc1|0": {
|
|
"em_relaxed": 0.011568123393316195,
|
|
"em_relaxed_stderr": 0.00383613885044431,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.011568123393316195,
|
|
"em_conservative_stderr": 0.00383613885044431
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:prehistory|0": {
|
|
"em_relaxed": 0.15047021943573669,
|
|
"em_relaxed_stderr": 0.020049383410796876,
|
|
"em": 0.04075235109717868,
|
|
"em_stderr": 0.011087359931149772,
|
|
"em_conservative": 0.14733542319749215,
|
|
"em_conservative_stderr": 0.019876006858615635
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:nutrition|0": {
|
|
"em_relaxed": 0.1848184818481848,
|
|
"em_relaxed_stderr": 0.022335536574954167,
|
|
"em": 0.036303630363036306,
|
|
"em_stderr": 0.010763206393659719,
|
|
"em_conservative": 0.1848184818481848,
|
|
"em_conservative_stderr": 0.022335536574954167
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0": {
|
|
"em_relaxed": 0.09090909090909091,
|
|
"em_relaxed_stderr": 0.020482086775424218,
|
|
"em": 0.005050505050505051,
|
|
"em_stderr": 0.00505050505050505,
|
|
"em_conservative": 0.08585858585858586,
|
|
"em_conservative_stderr": 0.01996022556317289
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0": {
|
|
"em_relaxed": 0.09361702127659574,
|
|
"em_relaxed_stderr": 0.01904256081095342,
|
|
"em": 0.03829787234042553,
|
|
"em_stderr": 0.012545838230269625,
|
|
"em_conservative": 0.09361702127659574,
|
|
"em_conservative_stderr": 0.01904256081095342
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0": {
|
|
"em_relaxed": 0.17,
|
|
"em_relaxed_stderr": 0.03775251680686371,
|
|
"em": 0.01,
|
|
"em_stderr": 0.009999999999999995,
|
|
"em_conservative": 0.12,
|
|
"em_conservative_stderr": 0.032659863237109066
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_medicine|0": {
|
|
"em_relaxed": 0.1569767441860465,
|
|
"em_relaxed_stderr": 0.027818863343493188,
|
|
"em": 0.040697674418604654,
|
|
"em_stderr": 0.01511000089316197,
|
|
"em_conservative": 0.1569767441860465,
|
|
"em_conservative_stderr": 0.027818863343493188
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_medicine|0": {
|
|
"em_relaxed": 0.20930232558139536,
|
|
"em_relaxed_stderr": 0.031109583909764628,
|
|
"em": 0.06395348837209303,
|
|
"em_stderr": 0.018710387348210598,
|
|
"em_conservative": 0.20930232558139536,
|
|
"em_conservative_stderr": 0.031109583909764628
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0": {
|
|
"em_relaxed": 0.18470149253731344,
|
|
"em_relaxed_stderr": 0.016777090030491517,
|
|
"em": 0.03731343283582089,
|
|
"em_stderr": 0.008194041665419557,
|
|
"em_conservative": 0.18470149253731344,
|
|
"em_conservative_stderr": 0.016777090030491517
|
|
},
|
|
"sklep.prompt2_nli_binary_sk:rte|0": {
|
|
"em_norm_relaxed": 0.3301204819277108,
|
|
"em_norm_relaxed_stderr": 0.011545464009906971,
|
|
"em_norm": 0.0018072289156626507,
|
|
"em_norm_stderr": 0.001042774975713993,
|
|
"em_conservative_norm": 0.3301204819277108,
|
|
"em_conservative_norm_stderr": 0.011545464009906971
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0": {
|
|
"em_relaxed": 0.14285714285714285,
|
|
"em_relaxed_stderr": 0.0238095238095238,
|
|
"em": 0.004608294930875576,
|
|
"em_stderr": 0.004608294930875571,
|
|
"em_conservative": 0.14285714285714285,
|
|
"em_conservative_stderr": 0.0238095238095238
|
|
},
|
|
"sklep.prompt2_hate_speech_sk:hate-speech|0": {
|
|
"em_norm_relaxed": 0.7247915087187263,
|
|
"em_norm_relaxed_stderr": 0.012302114305862654,
|
|
"em_norm": 0.0,
|
|
"em_norm_stderr": 0.0,
|
|
"em_conservative_norm": 0.7247915087187263,
|
|
"em_conservative_norm_stderr": 0.012302114305862654
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0": {
|
|
"em_relaxed": 0.13761467889908258,
|
|
"em_relaxed_stderr": 0.03314905716923366,
|
|
"em": 0.045871559633027525,
|
|
"em_stderr": 0.020130904850286974,
|
|
"em_conservative": 0.13761467889908258,
|
|
"em_conservative_stderr": 0.03314905716923366
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0": {
|
|
"em_relaxed": 0.22388059701492538,
|
|
"em_relaxed_stderr": 0.01802169384796668,
|
|
"em": 0.04477611940298507,
|
|
"em_stderr": 0.008941264053122127,
|
|
"em_conservative": 0.22388059701492538,
|
|
"em_conservative_stderr": 0.01802169384796668
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0": {
|
|
"em_relaxed": 0.19298245614035087,
|
|
"em_relaxed_stderr": 0.05273590328947574,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.19298245614035087,
|
|
"em_conservative_stderr": 0.05273590328947574
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:marketing|0": {
|
|
"em_relaxed": 0.1623931623931624,
|
|
"em_relaxed_stderr": 0.02416161812798774,
|
|
"em": 0.05555555555555555,
|
|
"em_stderr": 0.01500631280644691,
|
|
"em_conservative": 0.1623931623931624,
|
|
"em_conservative_stderr": 0.02416161812798774
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0": {
|
|
"em_relaxed": 0.17721518987341772,
|
|
"em_relaxed_stderr": 0.030475004517281346,
|
|
"em": 0.0379746835443038,
|
|
"em_stderr": 0.015254237576434418,
|
|
"em_conservative": 0.17721518987341772,
|
|
"em_conservative_stderr": 0.030475004517281346
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0": {
|
|
"em_relaxed": 0.14042553191489363,
|
|
"em_relaxed_stderr": 0.022712077616627854,
|
|
"em": 0.0851063829787234,
|
|
"em_stderr": 0.018241411344572286,
|
|
"em_conservative": 0.14042553191489363,
|
|
"em_conservative_stderr": 0.022712077616627854
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:virology|0": {
|
|
"em_relaxed": 0.16463414634146342,
|
|
"em_relaxed_stderr": 0.029047216079384597,
|
|
"em": 0.03048780487804878,
|
|
"em_stderr": 0.013466226472535378,
|
|
"em_conservative": 0.16463414634146342,
|
|
"em_conservative_stderr": 0.029047216079384597
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0": {
|
|
"em_relaxed": 0.14,
|
|
"em_relaxed_stderr": 0.034873508801977704,
|
|
"em": 0.01,
|
|
"em_stderr": 0.009999999999999997,
|
|
"em_conservative": 0.14,
|
|
"em_conservative_stderr": 0.034873508801977704
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0": {
|
|
"em_relaxed": 0.006675567423230975,
|
|
"em_relaxed_stderr": 0.0029774114441068625,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.0053404539385847796,
|
|
"em_conservative_stderr": 0.0026648668562206292
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0": {
|
|
"em_relaxed": 0.25252525252525254,
|
|
"em_relaxed_stderr": 0.043887175543277195,
|
|
"em": 0.030303030303030304,
|
|
"em_stderr": 0.01731601731601731,
|
|
"em_conservative": 0.24242424242424243,
|
|
"em_conservative_stderr": 0.04329004329004326
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:world_religions|0": {
|
|
"em_relaxed": 0.05263157894736842,
|
|
"em_relaxed_stderr": 0.017126088775157082,
|
|
"em": 0.023391812865497075,
|
|
"em_stderr": 0.01159224790573493,
|
|
"em_conservative": 0.05263157894736842,
|
|
"em_conservative_stderr": 0.017126088775157082
|
|
},
|
|
"senti_sk.prompt3_sentiment_binary_sk|0": {
|
|
"em_norm_relaxed": 0.25160944206008584,
|
|
"em_norm_relaxed_stderr": 0.0071080082101389235,
|
|
"em_norm": 0.03165236051502146,
|
|
"em_norm_stderr": 0.0028677349189843607,
|
|
"em_conservative_norm": 0.1990343347639485,
|
|
"em_conservative_norm_stderr": 0.0065402033880141534
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0": {
|
|
"em_relaxed": 0.13333333333333333,
|
|
"em_relaxed_stderr": 0.027848530325897826,
|
|
"em": 0.02666666666666667,
|
|
"em_stderr": 0.013198422611557032,
|
|
"em_conservative": 0.13333333333333333,
|
|
"em_conservative_stderr": 0.027848530325897826
|
|
},
|
|
"sklep.prompt1_nli_3_classes_sk:nli|0": {
|
|
"em_norm_relaxed": 0.016786570743405275,
|
|
"em_norm_relaxed_stderr": 0.0018163086111764766,
|
|
"em_norm": 0.0003996802557953637,
|
|
"em_norm_stderr": 0.00028258837305266234,
|
|
"em_conservative_norm": 0.016386890487609912,
|
|
"em_conservative_norm_stderr": 0.0017949203375471519
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0": {
|
|
"em_relaxed": 0.12919896640826872,
|
|
"em_relaxed_stderr": 0.017072429785340966,
|
|
"em": 0.012919896640826873,
|
|
"em_stderr": 0.005747937884708704,
|
|
"em_conservative": 0.12661498708010335,
|
|
"em_conservative_stderr": 0.016925900082204414
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0": {
|
|
"em_relaxed": 0.01335113484646195,
|
|
"em_relaxed_stderr": 0.004196522950335422,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.0053404539385847796,
|
|
"em_conservative_stderr": 0.0026648668562206297
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0": {
|
|
"em_relaxed": 0.11616161616161616,
|
|
"em_relaxed_stderr": 0.022828881775249377,
|
|
"em": 0.020202020202020204,
|
|
"em_stderr": 0.01002380379631872,
|
|
"em_conservative": 0.1111111111111111,
|
|
"em_conservative_stderr": 0.022390787638216763
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0": {
|
|
"em_relaxed": 0.32124352331606215,
|
|
"em_relaxed_stderr": 0.033699508685490674,
|
|
"em": 0.046632124352331605,
|
|
"em_stderr": 0.015216761819262577,
|
|
"em_conservative": 0.32124352331606215,
|
|
"em_conservative_stderr": 0.033699508685490674
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:business_ethics|0": {
|
|
"em_relaxed": 0.13,
|
|
"em_relaxed_stderr": 0.03379976689896308,
|
|
"em": 0.01,
|
|
"em_stderr": 0.009999999999999998,
|
|
"em_conservative": 0.13,
|
|
"em_conservative_stderr": 0.03379976689896308
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:public_relations|0": {
|
|
"em_relaxed": 0.23853211009174313,
|
|
"em_relaxed_stderr": 0.041009771029340336,
|
|
"em": 0.09174311926605505,
|
|
"em_stderr": 0.02777660875318803,
|
|
"em_conservative": 0.23853211009174313,
|
|
"em_conservative_stderr": 0.041009771029340336
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0": {
|
|
"em_relaxed": 0.16355140186915887,
|
|
"em_relaxed_stderr": 0.025342938086817392,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.1542056074766355,
|
|
"em_conservative_stderr": 0.024745297661074854
|
|
},
|
|
"senti_sk.prompt2_sentiment_binary_sk|0": {
|
|
"em_norm_relaxed": 0.4965128755364807,
|
|
"em_norm_relaxed_stderr": 0.008189921623320459,
|
|
"em_norm": 0.0037553648068669528,
|
|
"em_norm_stderr": 0.001001911513818721,
|
|
"em_conservative_norm": 0.49302575107296137,
|
|
"em_conservative_norm_stderr": 0.008189324033951761
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0": {
|
|
"em_relaxed": 0.18,
|
|
"em_relaxed_stderr": 0.03861229196653694,
|
|
"em": 0.07,
|
|
"em_stderr": 0.02564323999762429,
|
|
"em_conservative": 0.18,
|
|
"em_conservative_stderr": 0.03861229196653694
|
|
},
|
|
"belebele.prompt2_mqa_single_correct_sk|0": {
|
|
"em_relaxed": 0.2111111111111111,
|
|
"em_relaxed_stderr": 0.013610798969328482,
|
|
"em": 0.016666666666666666,
|
|
"em_stderr": 0.004269675894690937,
|
|
"em_conservative": 0.2111111111111111,
|
|
"em_conservative_stderr": 0.013610798969328482
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:virology|0": {
|
|
"em_relaxed": 0.12195121951219512,
|
|
"em_relaxed_stderr": 0.025630589418441155,
|
|
"em": 0.03048780487804878,
|
|
"em_stderr": 0.013466226472535378,
|
|
"em_conservative": 0.12195121951219512,
|
|
"em_conservative_stderr": 0.025630589418441155
|
|
},
|
|
"sklep.prompt2_nli_3_classes_sk:nli|0": {
|
|
"em_norm_relaxed": 0.09972022382094324,
|
|
"em_norm_relaxed_stderr": 0.004236089409386789,
|
|
"em_norm": 0.0,
|
|
"em_norm_stderr": 0.0,
|
|
"em_conservative_norm": 0.09952038369304557,
|
|
"em_conservative_norm_stderr": 0.004232312358957502
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0": {
|
|
"em_relaxed": 0.24444444444444444,
|
|
"em_relaxed_stderr": 0.03712537833614866,
|
|
"em": 0.014814814814814815,
|
|
"em_stderr": 0.010436494549594362,
|
|
"em_conservative": 0.24444444444444444,
|
|
"em_conservative_stderr": 0.03712537833614866
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0": {
|
|
"em_relaxed": 0.1731266149870801,
|
|
"em_relaxed_stderr": 0.01925784882518206,
|
|
"em": 0.031007751937984496,
|
|
"em_stderr": 0.00882270258926331,
|
|
"em_conservative": 0.17054263565891473,
|
|
"em_conservative_stderr": 0.019143434938568855
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0": {
|
|
"em_relaxed": 0.06766917293233082,
|
|
"em_relaxed_stderr": 0.015429701322264242,
|
|
"em": 0.011278195488721804,
|
|
"em_stderr": 0.0064868510822409155,
|
|
"em_conservative": 0.06766917293233082,
|
|
"em_conservative_stderr": 0.015429701322264242
|
|
},
|
|
"sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0": {
|
|
"em_norm_relaxed": 0.15355086372360843,
|
|
"em_norm_relaxed_stderr": 0.011173807886451477,
|
|
"em_norm": 0.015355086372360844,
|
|
"em_norm_stderr": 0.0038110143844746576,
|
|
"em_conservative_norm": 0.118042226487524,
|
|
"em_conservative_norm_stderr": 0.010000396690181516
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0": {
|
|
"em_relaxed": 0.1276595744680851,
|
|
"em_relaxed_stderr": 0.021815328994143183,
|
|
"em": 0.05106382978723404,
|
|
"em_stderr": 0.014390214367756187,
|
|
"em_conservative": 0.1276595744680851,
|
|
"em_conservative_stderr": 0.021815328994143183
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0": {
|
|
"em_relaxed": 0.12121212121212122,
|
|
"em_relaxed_stderr": 0.03296871474399962,
|
|
"em": 0.030303030303030304,
|
|
"em_stderr": 0.01731601731601732,
|
|
"em_conservative": 0.12121212121212122,
|
|
"em_conservative_stderr": 0.03296871474399962
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0": {
|
|
"em_relaxed": 0.20245398773006135,
|
|
"em_relaxed_stderr": 0.022289450459613447,
|
|
"em": 0.06134969325153374,
|
|
"em_stderr": 0.01331117614747688,
|
|
"em_conservative": 0.20245398773006135,
|
|
"em_conservative_stderr": 0.022289450459613447
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:security_studies|0": {
|
|
"em_relaxed": 0.1981981981981982,
|
|
"em_relaxed_stderr": 0.026815582053277207,
|
|
"em": 0.04504504504504504,
|
|
"em_stderr": 0.01395143304433832,
|
|
"em_conservative": 0.1981981981981982,
|
|
"em_conservative_stderr": 0.026815582053277207
|
|
},
|
|
"skj1.prompt1_skj1_sk|0": {
|
|
"em_relaxed": 0.02,
|
|
"em_relaxed_stderr": 0.014070529413628957,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.02,
|
|
"em_conservative_stderr": 0.014070529413628957
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_physics|0": {
|
|
"em_relaxed": 0.0784313725490196,
|
|
"em_relaxed_stderr": 0.026751495257385273,
|
|
"em": 0.00980392156862745,
|
|
"em_stderr": 0.009803921568627454,
|
|
"em_conservative": 0.0784313725490196,
|
|
"em_conservative_stderr": 0.026751495257385273
|
|
},
|
|
"sklep.prompt3_hate_speech_sk:hate-speech|0": {
|
|
"em_norm_relaxed": 0.6967399545109931,
|
|
"em_norm_relaxed_stderr": 0.012661502663418697,
|
|
"em_norm": 0.0,
|
|
"em_norm_stderr": 0.0,
|
|
"em_conservative_norm": 0.6967399545109931,
|
|
"em_conservative_norm_stderr": 0.012661502663418697
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0": {
|
|
"em_relaxed": 0.03759398496240601,
|
|
"em_relaxed_stderr": 0.011684641953170118,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.03759398496240601,
|
|
"em_conservative_stderr": 0.011684641953170118
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:virology|0": {
|
|
"em_relaxed": 0.1402439024390244,
|
|
"em_relaxed_stderr": 0.027197928271392544,
|
|
"em": 0.024390243902439025,
|
|
"em_stderr": 0.012082375722391943,
|
|
"em_conservative": 0.1402439024390244,
|
|
"em_conservative_stderr": 0.027197928271392544
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0": {
|
|
"em_relaxed": 0.05639097744360902,
|
|
"em_relaxed_stderr": 0.014170263190664145,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.05639097744360902,
|
|
"em_conservative_stderr": 0.014170263190664145
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:security_studies|0": {
|
|
"em_relaxed": 0.21621621621621623,
|
|
"em_relaxed_stderr": 0.02769147769493752,
|
|
"em": 0.04504504504504504,
|
|
"em_stderr": 0.013951433044338322,
|
|
"em_conservative": 0.21621621621621623,
|
|
"em_conservative_stderr": 0.02769147769493752
|
|
},
|
|
"financial_exams.prompt3_completion_sk|0": {
|
|
"em_relaxed": 0.24887556221889057,
|
|
"em_relaxed_stderr": 0.01184218927916494,
|
|
"em": 0.0037481259370314842,
|
|
"em_stderr": 0.0016736960391157028,
|
|
"em_conservative": 0.24887556221889057,
|
|
"em_conservative_stderr": 0.01184218927916494
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0": {
|
|
"em_relaxed": 0.18691588785046728,
|
|
"em_relaxed_stderr": 0.03786499587281739,
|
|
"em": 0.037383177570093455,
|
|
"em_stderr": 0.01842518672106723,
|
|
"em_conservative": 0.18691588785046728,
|
|
"em_conservative_stderr": 0.03786499587281739
|
|
},
|
|
"senti_sk.prompt3_sentiment_3_class_sk|0": {
|
|
"em_norm_relaxed": 0.40880906564036773,
|
|
"em_norm_relaxed_stderr": 0.0071893047031815266,
|
|
"em_norm": 0.10540945050245884,
|
|
"em_norm_stderr": 0.004490706548553942,
|
|
"em_conservative_norm": 0.26448578148385715,
|
|
"em_conservative_norm_stderr": 0.006449996068936491
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0": {
|
|
"em_relaxed": 0.1310344827586207,
|
|
"em_relaxed_stderr": 0.028119838672931184,
|
|
"em": 0.06896551724137931,
|
|
"em_stderr": 0.021116290886061897,
|
|
"em_conservative": 0.1310344827586207,
|
|
"em_conservative_stderr": 0.028119838672931184
|
|
},
|
|
"sklep.prompt3_qa_sk:question-answering|0": {
|
|
"exact_match_slk_prefix": 0.16143170197224252,
|
|
"exact_match_slk_prefix_stderr": 0.0037586816998114083,
|
|
"f1_slk": 0.3769392643579436,
|
|
"f1_slk_stderr": 0.003773157275405368
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0": {
|
|
"em_relaxed": 0.19630872483221476,
|
|
"em_relaxed_stderr": 0.016283806947934177,
|
|
"em": 0.018456375838926176,
|
|
"em_stderr": 0.005517845461546564,
|
|
"em_conservative": 0.19630872483221476,
|
|
"em_conservative_stderr": 0.016283806947934177
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:global_facts|0": {
|
|
"em_relaxed": 0.12,
|
|
"em_relaxed_stderr": 0.03265986323710906,
|
|
"em": 0.01,
|
|
"em_stderr": 0.009999999999999997,
|
|
"em_conservative": 0.12,
|
|
"em_conservative_stderr": 0.03265986323710906
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:computer_security|0": {
|
|
"em_relaxed": 0.11,
|
|
"em_relaxed_stderr": 0.03144660377352203,
|
|
"em": 0.02,
|
|
"em_stderr": 0.014070529413628963,
|
|
"em_conservative": 0.11,
|
|
"em_conservative_stderr": 0.03144660377352203
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:computer_security|0": {
|
|
"em_relaxed": 0.08,
|
|
"em_relaxed_stderr": 0.027265992434429082,
|
|
"em": 0.02,
|
|
"em_stderr": 0.014070529413628966,
|
|
"em_conservative": 0.08,
|
|
"em_conservative_stderr": 0.027265992434429082
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_physics|0": {
|
|
"em_relaxed": 0.12745098039215685,
|
|
"em_relaxed_stderr": 0.03318224921942078,
|
|
"em": 0.0392156862745098,
|
|
"em_stderr": 0.01931444240436096,
|
|
"em_conservative": 0.12745098039215685,
|
|
"em_conservative_stderr": 0.03318224921942078
|
|
},
|
|
"globalmmlu-lite.prompt2_mqa_single_correct_sk|0": {
|
|
"em_relaxed": 0.21,
|
|
"em_relaxed_stderr": 0.020390916450326858,
|
|
"em": 0.035,
|
|
"em_stderr": 0.009200495791370004,
|
|
"em_conservative": 0.21,
|
|
"em_conservative_stderr": 0.020390916450326858
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0": {
|
|
"em_relaxed": 0.26666666666666666,
|
|
"em_relaxed_stderr": 0.03305282343736876,
|
|
"em": 0.005555555555555556,
|
|
"em_stderr": 0.005555555555555554,
|
|
"em_conservative": 0.26666666666666666,
|
|
"em_conservative_stderr": 0.03305282343736876
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:anatomy|0": {
|
|
"em_relaxed": 0.13432835820895522,
|
|
"em_relaxed_stderr": 0.029568863219678807,
|
|
"em": 0.029850746268656716,
|
|
"em_stderr": 0.01475608176283378,
|
|
"em_conservative": 0.13432835820895522,
|
|
"em_conservative_stderr": 0.029568863219678807
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:security_studies|0": {
|
|
"em_relaxed": 0.15765765765765766,
|
|
"em_relaxed_stderr": 0.024513522045494097,
|
|
"em": 0.04954954954954955,
|
|
"em_stderr": 0.014597835283794243,
|
|
"em_conservative": 0.15765765765765766,
|
|
"em_conservative_stderr": 0.024513522045494097
|
|
},
|
|
"movie_review.prompt3_sentiment_binary_sk|0": {
|
|
"em_norm_relaxed": 0.11105577689243028,
|
|
"em_norm_relaxed_stderr": 0.007013489433691037,
|
|
"em_norm": 0.04731075697211155,
|
|
"em_norm_stderr": 0.004738947450976647,
|
|
"em_conservative_norm": 0.09412350597609562,
|
|
"em_conservative_norm_stderr": 0.006517932474820501
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0": {
|
|
"em_relaxed": 0.16,
|
|
"em_relaxed_stderr": 0.03684529491774709,
|
|
"em": 0.04,
|
|
"em_stderr": 0.01969463855669323,
|
|
"em_conservative": 0.16,
|
|
"em_conservative_stderr": 0.03684529491774709
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:astronomy|0": {
|
|
"em_relaxed": 0.17763157894736842,
|
|
"em_relaxed_stderr": 0.031103182383123398,
|
|
"em": 0.05263157894736842,
|
|
"em_stderr": 0.01817164224152954,
|
|
"em_conservative": 0.17105263157894737,
|
|
"em_conservative_stderr": 0.03064360707167709
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0": {
|
|
"em_relaxed": 0.18,
|
|
"em_relaxed_stderr": 0.038612291966536955,
|
|
"em": 0.04,
|
|
"em_stderr": 0.019694638556693223,
|
|
"em_conservative": 0.18,
|
|
"em_conservative_stderr": 0.038612291966536955
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0": {
|
|
"em_relaxed": 0.05,
|
|
"em_relaxed_stderr": 0.021904291355759026,
|
|
"em": 0.01,
|
|
"em_stderr": 0.009999999999999997,
|
|
"em_conservative": 0.04,
|
|
"em_conservative_stderr": 0.019694638556693237
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0": {
|
|
"em_relaxed": 0.0896551724137931,
|
|
"em_relaxed_stderr": 0.023807258824376967,
|
|
"em": 0.041379310344827586,
|
|
"em_stderr": 0.016597159859992713,
|
|
"em_conservative": 0.0896551724137931,
|
|
"em_conservative_stderr": 0.023807258824376967
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0": {
|
|
"em_relaxed": 0.2777777777777778,
|
|
"em_relaxed_stderr": 0.033477857599386325,
|
|
"em": 0.005555555555555556,
|
|
"em_stderr": 0.005555555555555546,
|
|
"em_conservative": 0.2777777777777778,
|
|
"em_conservative_stderr": 0.033477857599386325
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:philosophy|0": {
|
|
"em_relaxed": 0.237012987012987,
|
|
"em_relaxed_stderr": 0.024270317456654856,
|
|
"em": 0.05194805194805195,
|
|
"em_stderr": 0.01266576753511598,
|
|
"em_conservative": 0.237012987012987,
|
|
"em_conservative_stderr": 0.024270317456654856
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0": {
|
|
"em_relaxed": 0.21100917431192662,
|
|
"em_relaxed_stderr": 0.039262212380368526,
|
|
"em": 0.05504587155963303,
|
|
"em_stderr": 0.0219460246091855,
|
|
"em_conservative": 0.21100917431192662,
|
|
"em_conservative_stderr": 0.039262212380368526
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0": {
|
|
"em_relaxed": 0.01,
|
|
"em_relaxed_stderr": 0.009999999999999997,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.01,
|
|
"em_conservative_stderr": 0.009999999999999997
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:world_religions|0": {
|
|
"em_relaxed": 0.10526315789473684,
|
|
"em_relaxed_stderr": 0.02353755765789256,
|
|
"em": 0.04678362573099415,
|
|
"em_stderr": 0.01619638995623562,
|
|
"em_conservative": 0.10526315789473684,
|
|
"em_conservative_stderr": 0.02353755765789256
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0": {
|
|
"em_relaxed": 0.24324324324324326,
|
|
"em_relaxed_stderr": 0.026710910573068247,
|
|
"em": 0.04633204633204633,
|
|
"em_stderr": 0.013086679986931563,
|
|
"em_conservative": 0.23938223938223938,
|
|
"em_conservative_stderr": 0.026565582359277095
|
|
},
|
|
"toxic_sk.prompt1_toxicity_sk|0": {
|
|
"em_norm_relaxed": 0.45629965947786605,
|
|
"em_norm_relaxed_stderr": 0.016790496508382532,
|
|
"em_norm": 0.0,
|
|
"em_norm_stderr": 0.0,
|
|
"em_conservative_norm": 0.45629965947786605,
|
|
"em_conservative_norm_stderr": 0.016790496508382532
|
|
},
|
|
"senti_sk.prompt1_sentiment_3_class_sk|0": {
|
|
"em_norm_relaxed": 0.3200769724182168,
|
|
"em_norm_relaxed_stderr": 0.0068221276139399335,
|
|
"em_norm": 0.22642719692110327,
|
|
"em_norm_stderr": 0.006120369470744044,
|
|
"em_conservative_norm": 0.31922172332691895,
|
|
"em_conservative_norm_stderr": 0.006817290681927466
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_biology|0": {
|
|
"em_relaxed": 0.19014084507042253,
|
|
"em_relaxed_stderr": 0.03304706552420638,
|
|
"em": 0.04225352112676056,
|
|
"em_stderr": 0.01694131738100396,
|
|
"em_conservative": 0.19014084507042253,
|
|
"em_conservative_stderr": 0.03304706552420638
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:public_relations|0": {
|
|
"em_relaxed": 0.21100917431192662,
|
|
"em_relaxed_stderr": 0.03926221238036853,
|
|
"em": 0.07339449541284404,
|
|
"em_stderr": 0.025093850407933094,
|
|
"em_conservative": 0.21100917431192662,
|
|
"em_conservative_stderr": 0.03926221238036853
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0": {
|
|
"em_relaxed": 0.1969111969111969,
|
|
"em_relaxed_stderr": 0.024757519994692216,
|
|
"em": 0.03088803088803089,
|
|
"em_stderr": 0.010771402065873048,
|
|
"em_conservative": 0.1969111969111969,
|
|
"em_conservative_stderr": 0.024757519994692216
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:formal_logic|0": {
|
|
"em_relaxed": 0.13043478260869565,
|
|
"em_relaxed_stderr": 0.03154244565695702,
|
|
"em": 0.008695652173913044,
|
|
"em_stderr": 0.008695652173913031,
|
|
"em_conservative": 0.08695652173913043,
|
|
"em_conservative_stderr": 0.026390303420299282
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0": {
|
|
"em_relaxed": 0.20202020202020202,
|
|
"em_relaxed_stderr": 0.028606204289229872,
|
|
"em": 0.07575757575757576,
|
|
"em_stderr": 0.018852670234993107,
|
|
"em_conservative": 0.20202020202020202,
|
|
"em_conservative_stderr": 0.028606204289229872
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:prehistory|0": {
|
|
"em_relaxed": 0.15987460815047022,
|
|
"em_relaxed_stderr": 0.02055172319966096,
|
|
"em": 0.04075235109717868,
|
|
"em_stderr": 0.011087359931149773,
|
|
"em_conservative": 0.15987460815047022,
|
|
"em_conservative_stderr": 0.02055172319966096
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:econometrics|0": {
|
|
"em_relaxed": 0.125,
|
|
"em_relaxed_stderr": 0.03139045014587016,
|
|
"em": 0.026785714285714284,
|
|
"em_stderr": 0.015324784082343043,
|
|
"em_conservative": 0.125,
|
|
"em_conservative_stderr": 0.03139045014587016
|
|
},
|
|
"arc.prompt1_mqa_single_correct_sk|0": {
|
|
"em_relaxed": 0.1710863986313088,
|
|
"em_relaxed_stderr": 0.011018968811092407,
|
|
"em": 0.013686911890504704,
|
|
"em_stderr": 0.0033996852610928244,
|
|
"em_conservative": 0.1710863986313088,
|
|
"em_conservative_stderr": 0.011018968811092407
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:business_ethics|0": {
|
|
"em_relaxed": 0.14,
|
|
"em_relaxed_stderr": 0.03487350880197772,
|
|
"em": 0.02,
|
|
"em_stderr": 0.014070529413628957,
|
|
"em_conservative": 0.14,
|
|
"em_conservative_stderr": 0.03487350880197772
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0": {
|
|
"em_relaxed": 0.14018691588785046,
|
|
"em_relaxed_stderr": 0.03372118539369118,
|
|
"em": 0.056074766355140186,
|
|
"em_stderr": 0.02234599008672909,
|
|
"em_conservative": 0.14018691588785046,
|
|
"em_conservative_stderr": 0.03372118539369118
|
|
},
|
|
"truthfulqa.prompt2_mqa_single_correct_sk:mc1|0": {
|
|
"em_relaxed": 0.016709511568123392,
|
|
"em_relaxed_stderr": 0.004598458668423289,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.016709511568123392,
|
|
"em_conservative_stderr": 0.004598458668423289
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0": {
|
|
"em_relaxed": 0.14505776636713735,
|
|
"em_relaxed_stderr": 0.01262551435912769,
|
|
"em": 0.03209242618741977,
|
|
"em_stderr": 0.006318709962464618,
|
|
"em_conservative": 0.14505776636713735,
|
|
"em_conservative_stderr": 0.01262551435912769
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:astronomy|0": {
|
|
"em_relaxed": 0.14473684210526316,
|
|
"em_relaxed_stderr": 0.028631951845930412,
|
|
"em": 0.03289473684210526,
|
|
"em_stderr": 0.014514818445457586,
|
|
"em_conservative": 0.13815789473684212,
|
|
"em_conservative_stderr": 0.028081042939576552
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0": {
|
|
"em_relaxed": 0.21481481481481482,
|
|
"em_relaxed_stderr": 0.03547854198560824,
|
|
"em": 0.02962962962962963,
|
|
"em_stderr": 0.014648038602753806,
|
|
"em_conservative": 0.2074074074074074,
|
|
"em_conservative_stderr": 0.03502553170678318
|
|
},
|
|
"senti_sk.prompt2_sentiment_3_class_sk|0": {
|
|
"em_norm_relaxed": 0.383365405174257,
|
|
"em_norm_relaxed_stderr": 0.007110221850748201,
|
|
"em_norm": 0.019029292281376953,
|
|
"em_norm_stderr": 0.001998030377272694,
|
|
"em_conservative_norm": 0.37716484926234767,
|
|
"em_conservative_norm_stderr": 0.007087856422819165
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0": {
|
|
"em_relaxed": 0.23703703703703705,
|
|
"em_relaxed_stderr": 0.03673731683969506,
|
|
"em": 0.022222222222222223,
|
|
"em_stderr": 0.012733899715059683,
|
|
"em_conservative": 0.23703703703703705,
|
|
"em_conservative_stderr": 0.03673731683969506
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:formal_logic|0": {
|
|
"em_relaxed": 0.11304347826086956,
|
|
"em_relaxed_stderr": 0.029656609316837397,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.06956521739130435,
|
|
"em_conservative_stderr": 0.02382794649967283
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0": {
|
|
"em_relaxed": 0.30569948186528495,
|
|
"em_relaxed_stderr": 0.033248379397581594,
|
|
"em": 0.046632124352331605,
|
|
"em_stderr": 0.015216761819262604,
|
|
"em_conservative": 0.3005181347150259,
|
|
"em_conservative_stderr": 0.0330881859441575
|
|
},
|
|
"sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0": {
|
|
"em_norm_relaxed": 0.3512476007677543,
|
|
"em_norm_relaxed_stderr": 0.014795198864354053,
|
|
"em_norm": 0.0009596928982725527,
|
|
"em_norm_stderr": 0.0009596928982725573,
|
|
"em_conservative_norm": 0.34836852207293667,
|
|
"em_conservative_norm_stderr": 0.014767096602802959
|
|
},
|
|
"sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0": {
|
|
"em_norm_relaxed": 0.8809980806142035,
|
|
"em_norm_relaxed_stderr": 0.010035501945449067,
|
|
"em_norm": 0.44817658349328215,
|
|
"em_norm_stderr": 0.015413429449614385,
|
|
"em_conservative_norm": 0.8800383877159309,
|
|
"em_conservative_norm_stderr": 0.010070396971355563
|
|
},
|
|
"skj1.prompt2_skj1_sk|0": {
|
|
"em_relaxed": 0.06,
|
|
"em_relaxed_stderr": 0.023868325657594166,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.05,
|
|
"em_conservative_stderr": 0.021904291355759067
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:international_law|0": {
|
|
"em_relaxed": 0.16101694915254236,
|
|
"em_relaxed_stderr": 0.03397969368949715,
|
|
"em": 0.05084745762711865,
|
|
"em_stderr": 0.020309989475094187,
|
|
"em_conservative": 0.16101694915254236,
|
|
"em_conservative_stderr": 0.03397969368949715
|
|
},
|
|
"sklep.prompt1_nli_binary_sk:rte|0": {
|
|
"em_norm_relaxed": 0.030120481927710843,
|
|
"em_norm_relaxed_stderr": 0.004196301257202765,
|
|
"em_norm": 0.0006024096385542169,
|
|
"em_norm_stderr": 0.0006024096385542043,
|
|
"em_conservative_norm": 0.030120481927710843,
|
|
"em_conservative_norm_stderr": 0.004196301257202765
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:human_aging|0": {
|
|
"em_relaxed": 0.1278538812785388,
|
|
"em_relaxed_stderr": 0.02261639276396452,
|
|
"em": 0.0684931506849315,
|
|
"em_stderr": 0.017107571020219335,
|
|
"em_conservative": 0.1278538812785388,
|
|
"em_conservative_stderr": 0.02261639276396452
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:management|0": {
|
|
"em_relaxed": 0.2,
|
|
"em_relaxed_stderr": 0.040201512610368445,
|
|
"em": 0.06,
|
|
"em_stderr": 0.023868325657594162,
|
|
"em_conservative": 0.2,
|
|
"em_conservative_stderr": 0.040201512610368445
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0": {
|
|
"em_relaxed": 0.15217391304347827,
|
|
"em_relaxed_stderr": 0.02165993479655374,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.14130434782608695,
|
|
"em_conservative_stderr": 0.02100540442245809
|
|
},
|
|
"truthfulqa.prompt1_mqa_single_correct_sk:mc1|0": {
|
|
"em_relaxed": 0.007712082262210797,
|
|
"em_relaxed_stderr": 0.003138297919968842,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.007712082262210797,
|
|
"em_conservative_stderr": 0.003138297919968842
|
|
},
|
|
"senti_sk.prompt1_sentiment_binary_sk|0": {
|
|
"em_norm_relaxed": 0.31223175965665234,
|
|
"em_norm_relaxed_stderr": 0.0075906663188793026,
|
|
"em_norm": 0.1968884120171674,
|
|
"em_norm_stderr": 0.006513558648330005,
|
|
"em_conservative_norm": 0.3111587982832618,
|
|
"em_conservative_norm_stderr": 0.007583521183648285
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:astronomy|0": {
|
|
"em_relaxed": 0.1513157894736842,
|
|
"em_relaxed_stderr": 0.029162631596843985,
|
|
"em": 0.019736842105263157,
|
|
"em_stderr": 0.011319355505772411,
|
|
"em_conservative": 0.14473684210526316,
|
|
"em_conservative_stderr": 0.028631951845930412
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:human_aging|0": {
|
|
"em_relaxed": 0.1735159817351598,
|
|
"em_relaxed_stderr": 0.02564830822412722,
|
|
"em": 0.0867579908675799,
|
|
"em_stderr": 0.01906423736413682,
|
|
"em_conservative": 0.1735159817351598,
|
|
"em_conservative_stderr": 0.02564830822412722
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:formal_logic|0": {
|
|
"em_relaxed": 0.12173913043478261,
|
|
"em_relaxed_stderr": 0.030624883036764184,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.0782608695652174,
|
|
"em_conservative_stderr": 0.025154976812194616
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:sociology|0": {
|
|
"em_relaxed": 0.3350253807106599,
|
|
"em_relaxed_stderr": 0.03371423842235921,
|
|
"em": 0.05583756345177665,
|
|
"em_stderr": 0.016400555297895067,
|
|
"em_conservative": 0.3350253807106599,
|
|
"em_conservative_stderr": 0.03371423842235921
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:econometrics|0": {
|
|
"em_relaxed": 0.11607142857142858,
|
|
"em_relaxed_stderr": 0.03040253249894973,
|
|
"em": 0.017857142857142856,
|
|
"em_stderr": 0.012569900164490047,
|
|
"em_conservative": 0.11607142857142858,
|
|
"em_conservative_stderr": 0.03040253249894973
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0": {
|
|
"em_relaxed": 0.08847184986595175,
|
|
"em_relaxed_stderr": 0.014723671750091158,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.08579088471849866,
|
|
"em_conservative_stderr": 0.01452017587959949
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0": {
|
|
"em_relaxed": 0.06060606060606061,
|
|
"em_relaxed_stderr": 0.024102876029567216,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.050505050505050504,
|
|
"em_conservative_stderr": 0.02212079324207198
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0": {
|
|
"em_relaxed": 0.20134228187919462,
|
|
"em_relaxed_stderr": 0.016439528475054862,
|
|
"em": 0.03355704697986577,
|
|
"em_stderr": 0.0073828066874755535,
|
|
"em_conservative": 0.20134228187919462,
|
|
"em_conservative_stderr": 0.016439528475054862
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0": {
|
|
"em_relaxed": 0.09090909090909091,
|
|
"em_relaxed_stderr": 0.02903984386363362,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.08080808080808081,
|
|
"em_conservative_stderr": 0.027530712883605948
|
|
},
|
|
"globalmmlu-lite.prompt3_mqa_single_correct_sk|0": {
|
|
"em_relaxed": 0.2025,
|
|
"em_relaxed_stderr": 0.02011830610666578,
|
|
"em": 0.025,
|
|
"em_stderr": 0.007816023641463063,
|
|
"em_conservative": 0.2025,
|
|
"em_conservative_stderr": 0.02011830610666578
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0": {
|
|
"em_relaxed": 0.15762273901808785,
|
|
"em_relaxed_stderr": 0.018546804758294254,
|
|
"em": 0.01808785529715762,
|
|
"em_stderr": 0.006783224657656192,
|
|
"em_conservative": 0.14987080103359174,
|
|
"em_conservative_stderr": 0.018168008667456638
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0": {
|
|
"em_relaxed": 0.17050691244239632,
|
|
"em_relaxed_stderr": 0.025588834483805927,
|
|
"em": 0.013824884792626729,
|
|
"em_stderr": 0.007944762237164815,
|
|
"em_conservative": 0.16589861751152074,
|
|
"em_conservative_stderr": 0.025310686717794278
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0": {
|
|
"em_relaxed": 0.1559633027522936,
|
|
"em_relaxed_stderr": 0.03491242362801975,
|
|
"em": 0.05504587155963303,
|
|
"em_stderr": 0.021946024609185506,
|
|
"em_conservative": 0.1559633027522936,
|
|
"em_conservative_stderr": 0.03491242362801975
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:computer_security|0": {
|
|
"em_relaxed": 0.13,
|
|
"em_relaxed_stderr": 0.03379976689896309,
|
|
"em": 0.02,
|
|
"em_stderr": 0.014070529413628963,
|
|
"em_conservative": 0.13,
|
|
"em_conservative_stderr": 0.03379976689896309
|
|
},
|
|
"toxic_sk.prompt2_toxicity_sk|0": {
|
|
"em_norm_relaxed": 0.47559591373439275,
|
|
"em_norm_relaxed_stderr": 0.016834908286515442,
|
|
"em_norm": 0.0,
|
|
"em_norm_stderr": 0.0,
|
|
"em_conservative_norm": 0.47559591373439275,
|
|
"em_conservative_norm_stderr": 0.016834908286515442
|
|
},
|
|
"multi_wiki_qa.prompt3_qa_sk|0": {
|
|
"exact_match_slk_prefix": 0.34000400240144085,
|
|
"exact_match_slk_prefix_stderr": 0.006701954125609285,
|
|
"f1_slk": 0.5050071786470199,
|
|
"f1_slk_stderr": 0.005797838571943781
|
|
},
|
|
"financial_exams.prompt1_completion_sk|0": {
|
|
"em_relaxed": 0.20089955022488756,
|
|
"em_relaxed_stderr": 0.010974256206370726,
|
|
"em": 0.009745127436281859,
|
|
"em_stderr": 0.002690618851324012,
|
|
"em_conservative": 0.20089955022488756,
|
|
"em_conservative_stderr": 0.010974256206370726
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:philosophy|0": {
|
|
"em_relaxed": 0.21103896103896103,
|
|
"em_relaxed_stderr": 0.023288413887090093,
|
|
"em": 0.06818181818181818,
|
|
"em_stderr": 0.014385689805382164,
|
|
"em_conservative": 0.21103896103896103,
|
|
"em_conservative_stderr": 0.023288413887090093
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0": {
|
|
"em_relaxed": 0.2147239263803681,
|
|
"em_relaxed_stderr": 0.022777692313730433,
|
|
"em": 0.06748466257668712,
|
|
"em_stderr": 0.013915180741090785,
|
|
"em_conservative": 0.2116564417177914,
|
|
"em_conservative_stderr": 0.02265853497858724
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:marketing|0": {
|
|
"em_relaxed": 0.15384615384615385,
|
|
"em_relaxed_stderr": 0.023636873317489277,
|
|
"em": 0.03418803418803419,
|
|
"em_stderr": 0.011904341997629792,
|
|
"em_conservative": 0.15384615384615385,
|
|
"em_conservative_stderr": 0.023636873317489277
|
|
},
|
|
"globalmmlu-lite.prompt1_mqa_single_correct_sk|0": {
|
|
"em_relaxed": 0.1825,
|
|
"em_relaxed_stderr": 0.01933698975881026,
|
|
"em": 0.025,
|
|
"em_stderr": 0.007816023641463065,
|
|
"em_conservative": 0.1825,
|
|
"em_conservative_stderr": 0.01933698975881026
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_law|0": {
|
|
"em_relaxed": 0.12724167378309137,
|
|
"em_relaxed_stderr": 0.009742462799429212,
|
|
"em": 0.0017079419299743809,
|
|
"em_stderr": 0.0012071811002646103,
|
|
"em_conservative": 0.12724167378309137,
|
|
"em_conservative_stderr": 0.009742462799429212
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:machine_learning|0": {
|
|
"em_relaxed": 0.14285714285714285,
|
|
"em_relaxed_stderr": 0.033213611069662675,
|
|
"em": 0.026785714285714284,
|
|
"em_stderr": 0.015324784082343059,
|
|
"em_conservative": 0.125,
|
|
"em_conservative_stderr": 0.03139045014587016
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0": {
|
|
"em_relaxed": 0.2727272727272727,
|
|
"em_relaxed_stderr": 0.044988332664126655,
|
|
"em": 0.050505050505050504,
|
|
"em_stderr": 0.022120793242071984,
|
|
"em_conservative": 0.26262626262626265,
|
|
"em_conservative_stderr": 0.044452876769839444
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0": {
|
|
"em_relaxed": 0.30569948186528495,
|
|
"em_relaxed_stderr": 0.03324837939758159,
|
|
"em": 0.025906735751295335,
|
|
"em_stderr": 0.011464523356953171,
|
|
"em_conservative": 0.30569948186528495,
|
|
"em_conservative_stderr": 0.03324837939758159
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0": {
|
|
"em_relaxed": 0.17333333333333334,
|
|
"em_relaxed_stderr": 0.031010810941145068,
|
|
"em": 0.02666666666666667,
|
|
"em_stderr": 0.013198422611557013,
|
|
"em_conservative": 0.17333333333333334,
|
|
"em_conservative_stderr": 0.031010810941145068
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:management|0": {
|
|
"em_relaxed": 0.15,
|
|
"em_relaxed_stderr": 0.0358870281282637,
|
|
"em": 0.02,
|
|
"em_stderr": 0.01407052941362896,
|
|
"em_conservative": 0.15,
|
|
"em_conservative_stderr": 0.0358870281282637
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:sociology|0": {
|
|
"em_relaxed": 0.3096446700507614,
|
|
"em_relaxed_stderr": 0.03302479343505637,
|
|
"em": 0.050761421319796954,
|
|
"em_stderr": 0.015679295366526198,
|
|
"em_conservative": 0.3096446700507614,
|
|
"em_conservative_stderr": 0.03302479343505637
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0": {
|
|
"em_relaxed": 0.23232323232323232,
|
|
"em_relaxed_stderr": 0.04266016017054687,
|
|
"em": 0.030303030303030304,
|
|
"em_stderr": 0.01731601731601731,
|
|
"em_conservative": 0.2222222222222222,
|
|
"em_conservative_stderr": 0.041996052556580815
|
|
},
|
|
"skj1.prompt3_skj1_sk|0": {
|
|
"em_relaxed": 0.05,
|
|
"em_relaxed_stderr": 0.021904291355759067,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.03,
|
|
"em_conservative_stderr": 0.017144660799776543
|
|
},
|
|
"movie_review.prompt1_sentiment_binary_sk|0": {
|
|
"em_norm_relaxed": 0.6329681274900398,
|
|
"em_norm_relaxed_stderr": 0.010758931054934645,
|
|
"em_norm": 0.38745019920318724,
|
|
"em_norm_stderr": 0.010874391376667836,
|
|
"em_conservative_norm": 0.6324701195219123,
|
|
"em_conservative_norm_stderr": 0.010761991548945318
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:anatomy|0": {
|
|
"em_relaxed": 0.12686567164179105,
|
|
"em_relaxed_stderr": 0.028859365115889928,
|
|
"em": 0.022388059701492536,
|
|
"em_stderr": 0.012828198053329663,
|
|
"em_conservative": 0.12686567164179105,
|
|
"em_conservative_stderr": 0.028859365115889928
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0": {
|
|
"em_relaxed": 0.10101010101010101,
|
|
"em_relaxed_stderr": 0.030440148787487714,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.10101010101010101,
|
|
"em_conservative_stderr": 0.030440148787487714
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:philosophy|0": {
|
|
"em_relaxed": 0.2922077922077922,
|
|
"em_relaxed_stderr": 0.02595551182455085,
|
|
"em": 0.10064935064935066,
|
|
"em_stderr": 0.017171207138792156,
|
|
"em_conservative": 0.2922077922077922,
|
|
"em_conservative_stderr": 0.02595551182455085
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:human_aging|0": {
|
|
"em_relaxed": 0.1187214611872146,
|
|
"em_relaxed_stderr": 0.021907507389103187,
|
|
"em": 0.0365296803652968,
|
|
"em_stderr": 0.012706146532849703,
|
|
"em_conservative": 0.1187214611872146,
|
|
"em_conservative_stderr": 0.021907507389103187
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0": {
|
|
"em_relaxed": 0.10187667560321716,
|
|
"em_relaxed_stderr": 0.015683171805464465,
|
|
"em": 0.0160857908847185,
|
|
"em_stderr": 0.006522714286001283,
|
|
"em_conservative": 0.09651474530831099,
|
|
"em_conservative_stderr": 0.015310377502350168
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0": {
|
|
"em_relaxed": 0.13131313131313133,
|
|
"em_relaxed_stderr": 0.034117144062155554,
|
|
"em": 0.010101010101010102,
|
|
"em_stderr": 0.010101010101010109,
|
|
"em_conservative": 0.12121212121212122,
|
|
"em_conservative_stderr": 0.03296871474399963
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0": {
|
|
"em_relaxed": 0.13257575757575757,
|
|
"em_relaxed_stderr": 0.020910766968363862,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.13257575757575757,
|
|
"em_conservative_stderr": 0.020910766968363862
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0": {
|
|
"em_relaxed": 0.21308724832214765,
|
|
"em_relaxed_stderr": 0.01678740388759121,
|
|
"em": 0.050335570469798654,
|
|
"em_stderr": 0.008963220911107225,
|
|
"em_conservative": 0.21308724832214765,
|
|
"em_conservative_stderr": 0.01678740388759121
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0": {
|
|
"em_relaxed": 0.12413793103448276,
|
|
"em_relaxed_stderr": 0.0274782369836366,
|
|
"em": 0.041379310344827586,
|
|
"em_stderr": 0.016597159859992713,
|
|
"em_conservative": 0.12413793103448276,
|
|
"em_conservative_stderr": 0.0274782369836366
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0": {
|
|
"em_relaxed": 0.25316455696202533,
|
|
"em_relaxed_stderr": 0.0347027655481443,
|
|
"em": 0.05063291139240506,
|
|
"em_stderr": 0.017497810541168043,
|
|
"em_conservative": 0.25316455696202533,
|
|
"em_conservative_stderr": 0.0347027655481443
|
|
},
|
|
"sklep.prompt1_hate_speech_sk:hate-speech|0": {
|
|
"em_norm_relaxed": 0.6921910538286581,
|
|
"em_norm_relaxed_stderr": 0.012714401009923647,
|
|
"em_norm": 0.0,
|
|
"em_norm_stderr": 0.0,
|
|
"em_conservative_norm": 0.6921910538286581,
|
|
"em_conservative_norm_stderr": 0.012714401009923647
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0": {
|
|
"em_relaxed": 0.2092426187419769,
|
|
"em_relaxed_stderr": 0.014583336673745648,
|
|
"em": 0.0693196405648267,
|
|
"em_stderr": 0.009106230246473392,
|
|
"em_conservative": 0.2092426187419769,
|
|
"em_conservative_stderr": 0.014583336673745648
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:world_religions|0": {
|
|
"em_relaxed": 0.0935672514619883,
|
|
"em_relaxed_stderr": 0.022335993231163274,
|
|
"em": 0.04678362573099415,
|
|
"em_stderr": 0.016196389956235627,
|
|
"em_conservative": 0.0935672514619883,
|
|
"em_conservative_stderr": 0.022335993231163274
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0": {
|
|
"em_relaxed": 0.18439716312056736,
|
|
"em_relaxed_stderr": 0.023134670670309505,
|
|
"em": 0.02127659574468085,
|
|
"em_stderr": 0.00860850820920177,
|
|
"em_conservative": 0.1773049645390071,
|
|
"em_conservative_stderr": 0.022783829728850895
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0": {
|
|
"em_relaxed": 0.2777777777777778,
|
|
"em_relaxed_stderr": 0.03347785759938633,
|
|
"em": 0.005555555555555556,
|
|
"em_stderr": 0.005555555555555554,
|
|
"em_conservative": 0.2777777777777778,
|
|
"em_conservative_stderr": 0.03347785759938633
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_biology|0": {
|
|
"em_relaxed": 0.176056338028169,
|
|
"em_relaxed_stderr": 0.03207487957537854,
|
|
"em": 0.035211267605633804,
|
|
"em_stderr": 0.015521989489917001,
|
|
"em_conservative": 0.176056338028169,
|
|
"em_conservative_stderr": 0.03207487957537854
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:global_facts|0": {
|
|
"em_relaxed": 0.18,
|
|
"em_relaxed_stderr": 0.03861229196653694,
|
|
"em": 0.13,
|
|
"em_stderr": 0.03379976689896309,
|
|
"em_conservative": 0.18,
|
|
"em_conservative_stderr": 0.03861229196653694
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0": {
|
|
"em_relaxed": 0.25766871165644173,
|
|
"em_relaxed_stderr": 0.0242598463239315,
|
|
"em": 0.08895705521472393,
|
|
"em_stderr": 0.015791302137788298,
|
|
"em_conservative": 0.25766871165644173,
|
|
"em_conservative_stderr": 0.0242598463239315
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0": {
|
|
"em_relaxed": 0.15656565656565657,
|
|
"em_relaxed_stderr": 0.025890520358141454,
|
|
"em": 0.030303030303030304,
|
|
"em_stderr": 0.012213156893572809,
|
|
"em_conservative": 0.15656565656565657,
|
|
"em_conservative_stderr": 0.025890520358141454
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0": {
|
|
"em_relaxed": 0.0026702269692923898,
|
|
"em_relaxed_stderr": 0.001886873052453708,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.0026702269692923898,
|
|
"em_conservative_stderr": 0.001886873052453708
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_physics|0": {
|
|
"em_relaxed": 0.12745098039215685,
|
|
"em_relaxed_stderr": 0.03318224921942076,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.12745098039215685,
|
|
"em_conservative_stderr": 0.03318224921942076
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0": {
|
|
"em_relaxed": 0.09090909090909091,
|
|
"em_relaxed_stderr": 0.01772676309015986,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.09090909090909091,
|
|
"em_conservative_stderr": 0.01772676309015986
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0": {
|
|
"em_relaxed": 0.13924050632911392,
|
|
"em_relaxed_stderr": 0.027629554439252756,
|
|
"em": 0.02531645569620253,
|
|
"em_stderr": 0.012536706047327604,
|
|
"em_conservative": 0.13924050632911392,
|
|
"em_conservative_stderr": 0.027629554439252756
|
|
},
|
|
"sklep.prompt3_nli_binary_sk:rte|0": {
|
|
"em_norm_relaxed": 0.2319277108433735,
|
|
"em_norm_relaxed_stderr": 0.010362252629508847,
|
|
"em_norm": 0.0012048192771084338,
|
|
"em_norm_stderr": 0.0008516790803748009,
|
|
"em_conservative_norm": 0.2319277108433735,
|
|
"em_conservative_norm_stderr": 0.010362252629508847
|
|
},
|
|
"slovakcopa.prompt2_copa_sk|0": {
|
|
"em_relaxed": 0.51,
|
|
"em_relaxed_stderr": 0.02237859698923078,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.326,
|
|
"em_conservative_stderr": 0.020984009562393567
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0": {
|
|
"em_relaxed": 0.19626168224299065,
|
|
"em_relaxed_stderr": 0.038576441428227824,
|
|
"em": 0.056074766355140186,
|
|
"em_stderr": 0.02234599008672909,
|
|
"em_conservative": 0.19626168224299065,
|
|
"em_conservative_stderr": 0.038576441428227824
|
|
},
|
|
"multi_wiki_qa.prompt1_qa_sk|0": {
|
|
"exact_match_slk_prefix": 0.36401841104662797,
|
|
"exact_match_slk_prefix_stderr": 0.00680726616133214,
|
|
"f1_slk": 0.5156574509584114,
|
|
"f1_slk_stderr": 0.0058319674487835125
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0": {
|
|
"em_relaxed": 0.16791044776119404,
|
|
"em_relaxed_stderr": 0.016160210122502162,
|
|
"em": 0.016791044776119403,
|
|
"em_stderr": 0.005555010467354857,
|
|
"em_conservative": 0.16791044776119404,
|
|
"em_conservative_stderr": 0.016160210122502162
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:global_facts|0": {
|
|
"em_relaxed": 0.11,
|
|
"em_relaxed_stderr": 0.031446603773522035,
|
|
"em": 0.06,
|
|
"em_stderr": 0.02386832565759416,
|
|
"em_conservative": 0.11,
|
|
"em_conservative_stderr": 0.031446603773522035
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:sociology|0": {
|
|
"em_relaxed": 0.3553299492385787,
|
|
"em_relaxed_stderr": 0.034186655739878676,
|
|
"em": 0.07106598984771574,
|
|
"em_stderr": 0.018352502878671184,
|
|
"em_conservative": 0.3553299492385787,
|
|
"em_conservative_stderr": 0.034186655739878676
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0": {
|
|
"em_relaxed": 0.1950354609929078,
|
|
"em_relaxed_stderr": 0.02363698094391562,
|
|
"em": 0.02127659574468085,
|
|
"em_stderr": 0.00860850820920177,
|
|
"em_conservative": 0.1879432624113475,
|
|
"em_conservative_stderr": 0.023305230769714243
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:public_relations|0": {
|
|
"em_relaxed": 0.1743119266055046,
|
|
"em_relaxed_stderr": 0.03650561586028542,
|
|
"em": 0.027522935779816515,
|
|
"em_stderr": 0.015742553732396008,
|
|
"em_conservative": 0.1743119266055046,
|
|
"em_conservative_stderr": 0.03650561586028542
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0": {
|
|
"em_relaxed": 0.18691588785046728,
|
|
"em_relaxed_stderr": 0.026711670245826564,
|
|
"em": 0.004672897196261682,
|
|
"em_stderr": 0.004672897196261678,
|
|
"em_conservative": 0.17757009345794392,
|
|
"em_conservative_stderr": 0.02618451638732211
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:anatomy|0": {
|
|
"em_relaxed": 0.1791044776119403,
|
|
"em_relaxed_stderr": 0.03324844546213894,
|
|
"em": 0.05970149253731343,
|
|
"em_stderr": 0.020544692585423446,
|
|
"em_conservative": 0.1791044776119403,
|
|
"em_conservative_stderr": 0.03324844546213894
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:marketing|0": {
|
|
"em_relaxed": 0.2094017094017094,
|
|
"em_relaxed_stderr": 0.02665569965392276,
|
|
"em": 0.0811965811965812,
|
|
"em_stderr": 0.017893784904018422,
|
|
"em_conservative": 0.20512820512820512,
|
|
"em_conservative_stderr": 0.026453508054040356
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0": {
|
|
"em_relaxed": 0.14492753623188406,
|
|
"em_relaxed_stderr": 0.02122807279169736,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.13043478260869565,
|
|
"em_conservative_stderr": 0.020308668108561446
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0": {
|
|
"em_relaxed": 0.22727272727272727,
|
|
"em_relaxed_stderr": 0.02985751567338641,
|
|
"em": 0.0707070707070707,
|
|
"em_stderr": 0.01826310542019951,
|
|
"em_conservative": 0.22727272727272727,
|
|
"em_conservative_stderr": 0.02985751567338641
|
|
},
|
|
"slovakcopa.prompt1_copa_sk|0": {
|
|
"em_relaxed": 0.618,
|
|
"em_relaxed_stderr": 0.021750820591250834,
|
|
"em": 0.0,
|
|
"em_stderr": 0.0,
|
|
"em_conservative": 0.508,
|
|
"em_conservative_stderr": 0.022380208834928028
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:nutrition|0": {
|
|
"em_relaxed": 0.19801980198019803,
|
|
"em_relaxed_stderr": 0.02293151046482573,
|
|
"em": 0.0297029702970297,
|
|
"em_stderr": 0.00976897099052845,
|
|
"em_conservative": 0.19801980198019803,
|
|
"em_conservative_stderr": 0.02293151046482573
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:machine_learning|0": {
|
|
"em_relaxed": 0.125,
|
|
"em_relaxed_stderr": 0.03139045014587016,
|
|
"em": 0.008928571428571428,
|
|
"em_stderr": 0.008928571428571423,
|
|
"em_conservative": 0.11607142857142858,
|
|
"em_conservative_stderr": 0.03040253249894973
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:_average|0": {
|
|
"em_relaxed": 0.15750731608871743,
|
|
"em_relaxed_stderr": 0.027457423641973053,
|
|
"em": 0.028327141879067475,
|
|
"em_stderr": 0.010791129591908543,
|
|
"em_conservative": 0.15358971887332207,
|
|
"em_conservative_stderr": 0.027071657841089895
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:_average|0": {
|
|
"em_relaxed": 0.1431281879704793,
|
|
"em_relaxed_stderr": 0.02601762436348415,
|
|
"em": 0.023388116537651122,
|
|
"em_stderr": 0.010092739621009001,
|
|
"em_conservative": 0.14035369812713153,
|
|
"em_conservative_stderr": 0.02572013192933555
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:_average|0": {
|
|
"em_relaxed": 0.1796468718485245,
|
|
"em_relaxed_stderr": 0.02870969018742619,
|
|
"em": 0.04274500178923145,
|
|
"em_stderr": 0.01407239223594206,
|
|
"em_conservative": 0.17602310250998576,
|
|
"em_conservative_stderr": 0.028335799048817806
|
|
},
|
|
"all": {
|
|
"em_relaxed": 0.16276835973658582,
|
|
"em_relaxed_stderr": 0.02594967705725322,
|
|
"em": 0.029667227870516465,
|
|
"em_stderr": 0.010711849450086212,
|
|
"em_conservative": 0.15781564927252603,
|
|
"em_conservative_stderr": 0.025592515877326957,
|
|
"exact_match_slk_prefix": 0.25920345417449087,
|
|
"exact_match_slk_prefix_stderr": 0.005273401550208795,
|
|
"f1_slk": 0.4424731081623923,
|
|
"f1_slk_stderr": 0.004828570063081536,
|
|
"em_norm_relaxed": 0.37655055755239747,
|
|
"em_norm_relaxed_stderr": 0.009574346535835498,
|
|
"em_norm": 0.06270134241081571,
|
|
"em_norm_stderr": 0.0027847027649703544,
|
|
"em_conservative_norm": 0.3652434918007519,
|
|
"em_conservative_norm_stderr": 0.009448048478463963
|
|
}
|
|
},
|
|
"versions": {},
|
|
"config_tasks": {
|
|
"mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:miscellaneous",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"financial_exams.prompt2_completion_sk|0": {
|
|
"name": "financial_exams.prompt2_completion_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/slovak-financial-exam",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "7cdbfe635c3ed0d8b4df2ee04429d8afa651b33c",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:machine_learning|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:machine_learning",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"belebele.prompt3_mqa_single_correct_sk|0": {
|
|
"name": "belebele.prompt3_mqa_single_correct_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "facebook/belebele",
|
|
"hf_subset": "slk_Latn",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_law|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:professional_law",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:college_computer_science",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:professional_accounting",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_us_history",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_us_history",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_law|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:professional_law",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt1_qa_sk:question-answering|0": {
|
|
"name": "sklep.prompt1_qa_sk:question-answering",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "question-answering",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "exact_match_slk_prefix",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "f1_slk",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 126,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:college_chemistry",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:college_mathematics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_physics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_statistics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:professional_medicine",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"slovakcopa.prompt3_copa_sk|0": {
|
|
"name": "slovakcopa.prompt3_copa_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "NaiveNeuron/SlovakCOPA",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "5c61e4cd9343f9b4fab623bdc50bb895cc170771",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "validation",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"movie_review.prompt2_sentiment_binary_sk|0": {
|
|
"name": "movie_review.prompt2_sentiment_binary_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "mteb/slovak_movie_review_sentiment",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "29a7405aabcfd4860a51ae6f65a5650d63108f26",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt2_qa_sk:question-answering|0": {
|
|
"name": "sklep.prompt2_qa_sk:question-answering",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "question-answering",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "exact_match_slk_prefix",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "f1_slk",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 126,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_biology|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:college_biology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:nutrition|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:nutrition",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_biology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"arc.prompt3_mqa_single_correct_sk|0": {
|
|
"name": "arc.prompt3_mqa_single_correct_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "jon-tow/okapi_arc_challenge",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": [
|
|
"solve"
|
|
],
|
|
"scorer": "score",
|
|
"sample_fields": "record_to_sample",
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "56aa31aedeb55f0635cc7c3c13f7f86b276f9a4e",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:econometrics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:econometrics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:prehistory|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:prehistory",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"toxic_sk.prompt3_toxicity_sk|0": {
|
|
"name": "toxic_sk.prompt3_toxicity_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/toxic-sk",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "2fb416e18785c128e7ec934c07a14c803a88cdb3",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"belebele.prompt1_mqa_single_correct_sk|0": {
|
|
"name": "belebele.prompt1_mqa_single_correct_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "facebook/belebele",
|
|
"hf_subset": "slk_Latn",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"arc.prompt2_mqa_single_correct_sk|0": {
|
|
"name": "arc.prompt2_mqa_single_correct_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "jon-tow/okapi_arc_challenge",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": [
|
|
"solve"
|
|
],
|
|
"scorer": "score",
|
|
"sample_fields": "record_to_sample",
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "56aa31aedeb55f0635cc7c3c13f7f86b276f9a4e",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"multi_wiki_qa.prompt2_qa_sk|0": {
|
|
"name": "multi_wiki_qa.prompt2_qa_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "alexandrainst/multi-wiki-qa",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "exact_match_slk_prefix",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "f1_slk",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "9bdb3350487e399bd1cd873e1395290d636db5ce",
|
|
"hf_filter": "<lambda>",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"train"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 126,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:management|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:management",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:abstract_algebra",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:college_chemistry",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt3_nli_3_classes_sk:nli|0": {
|
|
"name": "sklep.prompt3_nli_3_classes_sk:nli",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "nli",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_medicine|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:college_medicine",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:business_ethics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:business_ethics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:college_computer_science",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:international_law|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:international_law",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:international_law|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:international_law",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"truthfulqa.prompt3_mqa_single_correct_sk:mc1|0": {
|
|
"name": "truthfulqa.prompt3_mqa_single_correct_sk:mc1",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "alexandrainst/m_truthfulqa",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "f0445d470f1925882b990f5f247fdcf288972f60",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"val"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:prehistory|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:prehistory",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:nutrition|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:nutrition",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:conceptual_physics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:college_computer_science",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_medicine|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:college_medicine",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_medicine|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:college_medicine",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_psychology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt2_nli_binary_sk:rte|0": {
|
|
"name": "sklep.prompt2_nli_binary_sk:rte",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "rte",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt2_hate_speech_sk:hate-speech|0": {
|
|
"name": "sklep.prompt2_hate_speech_sk:hate-speech",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "hate-speech",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:human_sexuality",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_psychology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_us_history",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:marketing|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:marketing",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:logical_fallacies",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:conceptual_physics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:virology|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:virology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:medical_genetics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:moral_scenarios",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:world_religions|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:world_religions",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"senti_sk.prompt3_sentiment_binary_sk|0": {
|
|
"name": "senti_sk.prompt3_sentiment_binary_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/senti-sk",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d",
|
|
"hf_filter": "<lambda>",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_physics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt1_nli_3_classes_sk:nli|0": {
|
|
"name": "sklep.prompt1_nli_3_classes_sk:nli",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "nli",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:moral_scenarios",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:business_ethics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:business_ethics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:public_relations|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:public_relations",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_statistics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"senti_sk.prompt2_sentiment_binary_sk|0": {
|
|
"name": "senti_sk.prompt2_sentiment_binary_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/senti-sk",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d",
|
|
"hf_filter": "<lambda>",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:medical_genetics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"belebele.prompt2_mqa_single_correct_sk|0": {
|
|
"name": "belebele.prompt2_mqa_single_correct_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "facebook/belebele",
|
|
"hf_subset": "slk_Latn",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:virology|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:virology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt2_nli_3_classes_sk:nli|0": {
|
|
"name": "sklep.prompt2_nli_3_classes_sk:nli",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "nli",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_european_history",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0": {
|
|
"name": "sklep.prompt3_sentiment_binary_sk:sentiment-analysis",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "sentiment-analysis",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:conceptual_physics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:moral_disputes",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:security_studies|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:security_studies",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"skj1.prompt1_skj1_sk|0": {
|
|
"name": "skj1.prompt1_skj1_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/SKJ1",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "e615c60b415389b629703f82d3a79810dbde6734",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train"
|
|
],
|
|
"evaluation_splits": [
|
|
"train"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_physics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:college_physics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt3_hate_speech_sk:hate-speech|0": {
|
|
"name": "sklep.prompt3_hate_speech_sk:hate-speech",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "hate-speech",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:virology|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:virology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:security_studies|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:security_studies",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"financial_exams.prompt3_completion_sk|0": {
|
|
"name": "financial_exams.prompt3_completion_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/slovak-financial-exam",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "7cdbfe635c3ed0d8b4df2ee04429d8afa651b33c",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:jurisprudence",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"senti_sk.prompt3_sentiment_3_class_sk|0": {
|
|
"name": "senti_sk.prompt3_sentiment_3_class_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/senti-sk",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:electrical_engineering",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt3_qa_sk:question-answering|0": {
|
|
"name": "sklep.prompt3_qa_sk:question-answering",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "question-answering",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "exact_match_slk_prefix",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "f1_slk",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 126,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:professional_psychology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:global_facts|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:global_facts",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:computer_security|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:computer_security",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:computer_security|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:computer_security",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_physics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:college_physics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"globalmmlu-lite.prompt2_mqa_single_correct_sk|0": {
|
|
"name": "globalmmlu-lite.prompt2_mqa_single_correct_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "CohereLabs/Global-MMLU-Lite",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "600af1bff2d447c7fc3aee22f11f607560d5fcb7",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": "1"
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_world_history",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:anatomy|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:anatomy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:security_studies|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:security_studies",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"movie_review.prompt3_sentiment_binary_sk|0": {
|
|
"name": "movie_review.prompt3_sentiment_binary_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "mteb/slovak_movie_review_sentiment",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "29a7405aabcfd4860a51ae6f65a5650d63108f26",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:college_chemistry",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:astronomy|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:astronomy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:medical_genetics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:abstract_algebra",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:electrical_engineering",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_world_history",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:philosophy|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:philosophy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:human_sexuality",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:abstract_algebra",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:world_religions|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:world_religions",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"toxic_sk.prompt1_toxicity_sk|0": {
|
|
"name": "toxic_sk.prompt1_toxicity_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/toxic-sk",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "2fb416e18785c128e7ec934c07a14c803a88cdb3",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"senti_sk.prompt1_sentiment_3_class_sk|0": {
|
|
"name": "senti_sk.prompt1_sentiment_3_class_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/senti-sk",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_biology|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:college_biology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:public_relations|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:public_relations",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:formal_logic|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:formal_logic",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_geography",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:prehistory|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:prehistory",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:econometrics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:econometrics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"arc.prompt1_mqa_single_correct_sk|0": {
|
|
"name": "arc.prompt1_mqa_single_correct_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "jon-tow/okapi_arc_challenge",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": [
|
|
"solve"
|
|
],
|
|
"scorer": "score",
|
|
"sample_fields": "record_to_sample",
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "56aa31aedeb55f0635cc7c3c13f7f86b276f9a4e",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:business_ethics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:business_ethics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:jurisprudence",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"truthfulqa.prompt2_mqa_single_correct_sk:mc1|0": {
|
|
"name": "truthfulqa.prompt2_mqa_single_correct_sk:mc1",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "alexandrainst/m_truthfulqa",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "f0445d470f1925882b990f5f247fdcf288972f60",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"val"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:miscellaneous",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:astronomy|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:astronomy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_european_history",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"senti_sk.prompt2_sentiment_3_class_sk|0": {
|
|
"name": "senti_sk.prompt2_sentiment_3_class_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/senti-sk",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_european_history",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:formal_logic|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:formal_logic",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0": {
|
|
"name": "sklep.prompt2_sentiment_binary_sk:sentiment-analysis",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "sentiment-analysis",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0": {
|
|
"name": "sklep.prompt1_sentiment_binary_sk:sentiment-analysis",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "sentiment-analysis",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"skj1.prompt2_skj1_sk|0": {
|
|
"name": "skj1.prompt2_skj1_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/SKJ1",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "e615c60b415389b629703f82d3a79810dbde6734",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train"
|
|
],
|
|
"evaluation_splits": [
|
|
"train"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:international_law|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:international_law",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt1_nli_binary_sk:rte|0": {
|
|
"name": "sklep.prompt1_nli_binary_sk:rte",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "rte",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:human_aging|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:human_aging",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:management|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:management",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:professional_accounting",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"truthfulqa.prompt1_mqa_single_correct_sk:mc1|0": {
|
|
"name": "truthfulqa.prompt1_mqa_single_correct_sk:mc1",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "alexandrainst/m_truthfulqa",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "f0445d470f1925882b990f5f247fdcf288972f60",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"val"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"senti_sk.prompt1_sentiment_binary_sk|0": {
|
|
"name": "senti_sk.prompt1_sentiment_binary_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/senti-sk",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d",
|
|
"hf_filter": "<lambda>",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:astronomy|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:astronomy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:human_aging|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:human_aging",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:formal_logic|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:formal_logic",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:sociology|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:sociology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:econometrics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:econometrics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:college_mathematics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:professional_psychology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:college_mathematics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"globalmmlu-lite.prompt3_mqa_single_correct_sk|0": {
|
|
"name": "globalmmlu-lite.prompt3_mqa_single_correct_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "CohereLabs/Global-MMLU-Lite",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "600af1bff2d447c7fc3aee22f11f607560d5fcb7",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": "1"
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:human_sexuality",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:computer_security|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:computer_security",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"toxic_sk.prompt2_toxicity_sk|0": {
|
|
"name": "toxic_sk.prompt2_toxicity_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/toxic-sk",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "2fb416e18785c128e7ec934c07a14c803a88cdb3",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"multi_wiki_qa.prompt3_qa_sk|0": {
|
|
"name": "multi_wiki_qa.prompt3_qa_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "alexandrainst/multi-wiki-qa",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "exact_match_slk_prefix",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "f1_slk",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "9bdb3350487e399bd1cd873e1395290d636db5ce",
|
|
"hf_filter": "<lambda>",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"train"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 126,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"financial_exams.prompt1_completion_sk|0": {
|
|
"name": "financial_exams.prompt1_completion_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/slovak-financial-exam",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "7cdbfe635c3ed0d8b4df2ee04429d8afa651b33c",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:philosophy|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:philosophy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:moral_disputes",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:marketing|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:marketing",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"globalmmlu-lite.prompt1_mqa_single_correct_sk|0": {
|
|
"name": "globalmmlu-lite.prompt1_mqa_single_correct_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "CohereLabs/Global-MMLU-Lite",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "600af1bff2d447c7fc3aee22f11f607560d5fcb7",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": "1"
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_law|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:professional_law",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:machine_learning|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:machine_learning",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_physics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:management|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:management",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:sociology|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:sociology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"skj1.prompt3_skj1_sk|0": {
|
|
"name": "skj1.prompt3_skj1_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/SKJ1",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "e615c60b415389b629703f82d3a79810dbde6734",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train"
|
|
],
|
|
"evaluation_splits": [
|
|
"train"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"movie_review.prompt1_sentiment_binary_sk|0": {
|
|
"name": "movie_review.prompt1_sentiment_binary_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "mteb/slovak_movie_review_sentiment",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "29a7405aabcfd4860a51ae6f65a5650d63108f26",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:anatomy|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:anatomy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:philosophy|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:philosophy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:human_aging|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:human_aging",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:professional_medicine",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:professional_psychology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:electrical_engineering",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:logical_fallacies",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt1_hate_speech_sk:hate-speech|0": {
|
|
"name": "sklep.prompt1_hate_speech_sk:hate-speech",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "hate-speech",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:miscellaneous",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:world_religions|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:world_religions",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_biology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_world_history",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_biology|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:college_biology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:global_facts|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:global_facts",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:moral_disputes",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_geography",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:moral_scenarios",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_physics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:college_physics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:professional_medicine",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:logical_fallacies",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt3_nli_binary_sk:rte|0": {
|
|
"name": "sklep.prompt3_nli_binary_sk:rte",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "rte",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"slovakcopa.prompt2_copa_sk|0": {
|
|
"name": "slovakcopa.prompt2_copa_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "NaiveNeuron/SlovakCOPA",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "5c61e4cd9343f9b4fab623bdc50bb895cc170771",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "validation",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:jurisprudence",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"multi_wiki_qa.prompt1_qa_sk|0": {
|
|
"name": "multi_wiki_qa.prompt1_qa_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "alexandrainst/multi-wiki-qa",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "exact_match_slk_prefix",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "f1_slk",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "9bdb3350487e399bd1cd873e1395290d636db5ce",
|
|
"hf_filter": "<lambda>",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"train"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 126,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_psychology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:global_facts|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:global_facts",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:sociology|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:sociology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_biology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:public_relations|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:public_relations",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_statistics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:anatomy|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:anatomy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:marketing|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:marketing",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:professional_accounting",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_geography",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"slovakcopa.prompt1_copa_sk|0": {
|
|
"name": "slovakcopa.prompt1_copa_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "NaiveNeuron/SlovakCOPA",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "5c61e4cd9343f9b4fab623bdc50bb895cc170771",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "validation",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:nutrition|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:nutrition",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:machine_learning|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:machine_learning",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
}
|
|
},
|
|
"summary_tasks": {
|
|
"mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0": {
|
|
"hashes": {
|
|
"hash_examples": "81792091d0a281a1",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "197fc777a54fa451",
|
|
"hash_cont_tokens": "2ca1254365bc4de8"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0": {
|
|
"hashes": {
|
|
"hash_examples": "ac8d3b0039dcb239",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "5db4f751889ea833",
|
|
"hash_cont_tokens": "e63825b594496705"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"financial_exams.prompt2_completion_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "298560bc2b080ed4",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "0342f86b74762ed2",
|
|
"hash_cont_tokens": "b8fcb2be5fc35e79"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:machine_learning|0": {
|
|
"hashes": {
|
|
"hash_examples": "9c6bb044db076a06",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "9d92fa8ac1790956",
|
|
"hash_cont_tokens": "13ea8227d1f89b7a"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"belebele.prompt3_mqa_single_correct_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "40ce6eca2db6a52f",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "6461c4c38cc687cc",
|
|
"hash_cont_tokens": "117c4c7e7c956a19"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0": {
|
|
"hashes": {
|
|
"hash_examples": "f7fbee8f9cf664d7",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "5f16ff9acfd0b051",
|
|
"hash_cont_tokens": "dc758eff6372a6bd"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_law|0": {
|
|
"hashes": {
|
|
"hash_examples": "645c5fc320444ab9",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "4d8fda65be8ecb47",
|
|
"hash_cont_tokens": "680d5153c8206395"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0": {
|
|
"hashes": {
|
|
"hash_examples": "f2818986ee796bdd",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "c117c2660e917e7b",
|
|
"hash_cont_tokens": "f070535f9aa37264"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0": {
|
|
"hashes": {
|
|
"hash_examples": "ce1327e99680eaa4",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "64ce9a865c4933e8",
|
|
"hash_cont_tokens": "b0cad34573a530c2"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0": {
|
|
"hashes": {
|
|
"hash_examples": "648884be05478291",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "c306237952459a8c",
|
|
"hash_cont_tokens": "0d3b3fe8f6a6e93e"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0": {
|
|
"hashes": {
|
|
"hash_examples": "ea7b1315dcf37a69",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "af88757d442d46fb",
|
|
"hash_cont_tokens": "2a0c5d184ed08ac7"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0": {
|
|
"hashes": {
|
|
"hash_examples": "648884be05478291",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "dc75ba147ed27b16",
|
|
"hash_cont_tokens": "a112819f21ae3613"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_law|0": {
|
|
"hashes": {
|
|
"hash_examples": "645c5fc320444ab9",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "2ad645d70202f61c",
|
|
"hash_cont_tokens": "568fc39fd238e02c"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"sklep.prompt1_qa_sk:question-answering|0": {
|
|
"hashes": {
|
|
"hash_examples": "8026b7009fd8d93f",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "9b2ecc66440c6655",
|
|
"hash_cont_tokens": "7b155235596271e5"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0": {
|
|
"hashes": {
|
|
"hash_examples": "fc35facc4060af81",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "5019761d5bacf09d",
|
|
"hash_cont_tokens": "22c11b68f032f39c"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0": {
|
|
"hashes": {
|
|
"hash_examples": "2e0d5171d0d40ca2",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "ed2271f4780c6c74",
|
|
"hash_cont_tokens": "bdc0ee0b2faaa2e8"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0": {
|
|
"hashes": {
|
|
"hash_examples": "5bc6e3e3c9ba587f",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "3042c9a7af42d006",
|
|
"hash_cont_tokens": "a86bd0a20a7d22a5"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0": {
|
|
"hashes": {
|
|
"hash_examples": "992e1937e8f9557c",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "0210a9fbd544fe47",
|
|
"hash_cont_tokens": "e552bd0023154eb6"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0": {
|
|
"hashes": {
|
|
"hash_examples": "00f9ec3430780b27",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "7c85e86631c58d50",
|
|
"hash_cont_tokens": "cd04bbd91a899720"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0": {
|
|
"hashes": {
|
|
"hash_examples": "ea73340f4d960eee",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "8973856ba2b7c3e4",
|
|
"hash_cont_tokens": "73211ae3847f43b5"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"slovakcopa.prompt3_copa_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "19e8e1315421e50c",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "c14d1e258331eaae",
|
|
"hash_cont_tokens": "4d37bc2a9714402e"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"movie_review.prompt2_sentiment_binary_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "f731da94d9560ab2",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "eda480540fb1d6c4",
|
|
"hash_cont_tokens": "5ff999ac05d6ebee"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"sklep.prompt2_qa_sk:question-answering|0": {
|
|
"hashes": {
|
|
"hash_examples": "8026b7009fd8d93f",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "805c2f8789ee2559",
|
|
"hash_cont_tokens": "b6a58ddf979e8542"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_biology|0": {
|
|
"hashes": {
|
|
"hash_examples": "8bd34873c0ac9972",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "ba08fe18dcb39b17",
|
|
"hash_cont_tokens": "1c0532ee760fc3d2"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:nutrition|0": {
|
|
"hashes": {
|
|
"hash_examples": "47563eff5f4089bf",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "e5398ee043cbef07",
|
|
"hash_cont_tokens": "0392303738faba09"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0": {
|
|
"hashes": {
|
|
"hash_examples": "27b6684dc6c0d1b3",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "8d522cc6cead753a",
|
|
"hash_cont_tokens": "0ff90fe9e9aefbc0"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"arc.prompt3_mqa_single_correct_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "6725e9266ede5f20",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "b9a63364b1f284dd",
|
|
"hash_cont_tokens": "15b124076d60386a"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:econometrics|0": {
|
|
"hashes": {
|
|
"hash_examples": "7d90b1f77d986bbe",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "db859ef4b587dde8",
|
|
"hash_cont_tokens": "e774cef893436bb0"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:prehistory|0": {
|
|
"hashes": {
|
|
"hash_examples": "375708e013c71363",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "108f91cedf2751e3",
|
|
"hash_cont_tokens": "e300daa822554479"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"toxic_sk.prompt3_toxicity_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "d01cdad28edb6cc2",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "b572324ccee144a3",
|
|
"hash_cont_tokens": "894f32af192a6aba"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"belebele.prompt1_mqa_single_correct_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "40ce6eca2db6a52f",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "568128be863811c3",
|
|
"hash_cont_tokens": "926371267f4e5206"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"arc.prompt2_mqa_single_correct_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "6725e9266ede5f20",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "7b3aca5e97b189fd",
|
|
"hash_cont_tokens": "fecd44e229b2e8b5"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"multi_wiki_qa.prompt2_qa_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "f166d5d501f82d9e",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "5e1bd4b50a6cd47e",
|
|
"hash_cont_tokens": "60f3560556af052d"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:management|0": {
|
|
"hashes": {
|
|
"hash_examples": "6be6db70af55c301",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "222ff13fe0318e8d",
|
|
"hash_cont_tokens": "7a5047bcd8b04f08"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0": {
|
|
"hashes": {
|
|
"hash_examples": "1b29eb003b1f4bae",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "9ee3fdc1e5b7dafb",
|
|
"hash_cont_tokens": "5ade4ebf51fa62e1"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0": {
|
|
"hashes": {
|
|
"hash_examples": "fc35facc4060af81",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "39550d80d1a77e43",
|
|
"hash_cont_tokens": "5b1778c721f6f013"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"sklep.prompt3_nli_3_classes_sk:nli|0": {
|
|
"hashes": {
|
|
"hash_examples": "4ba8c3712f2beb9b",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "93b6e471405ec4a2",
|
|
"hash_cont_tokens": "b00e20d4002c0576"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_medicine|0": {
|
|
"hashes": {
|
|
"hash_examples": "bfbac8bb6ff2adc7",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "90abb1fe093d34b9",
|
|
"hash_cont_tokens": "86ebffd8f5e2f04a"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:business_ethics|0": {
|
|
"hashes": {
|
|
"hash_examples": "f891ffad3913452c",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "313df26edb3e563a",
|
|
"hash_cont_tokens": "50713a1395a44b0a"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0": {
|
|
"hashes": {
|
|
"hash_examples": "f2818986ee796bdd",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "be7503adf6970be2",
|
|
"hash_cont_tokens": "122abbf33ae2f3ff"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:international_law|0": {
|
|
"hashes": {
|
|
"hash_examples": "a3aec6cd58af642c",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "bf45b63fa78edc41",
|
|
"hash_cont_tokens": "26ef649f0f6b3ee1"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:international_law|0": {
|
|
"hashes": {
|
|
"hash_examples": "a3aec6cd58af642c",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "ec8c1433297b3b71",
|
|
"hash_cont_tokens": "0a6070b89f5afa81"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"truthfulqa.prompt3_mqa_single_correct_sk:mc1|0": {
|
|
"hashes": {
|
|
"hash_examples": "ee751399d6f4ae5f",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "cd647a82dbdd5bc7",
|
|
"hash_cont_tokens": "69cd33952c06db63"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:prehistory|0": {
|
|
"hashes": {
|
|
"hash_examples": "375708e013c71363",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "08f1d64053a47ae0",
|
|
"hash_cont_tokens": "58f83cb29db5c299"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:nutrition|0": {
|
|
"hashes": {
|
|
"hash_examples": "47563eff5f4089bf",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "c33f74fed807aefd",
|
|
"hash_cont_tokens": "fd939458a7f74016"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0": {
|
|
"hashes": {
|
|
"hash_examples": "ea7b1315dcf37a69",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "3c3de4494a17e944",
|
|
"hash_cont_tokens": "5eb735aace25626b"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0": {
|
|
"hashes": {
|
|
"hash_examples": "7af87a7ae1cb0e53",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "1246824958e77888",
|
|
"hash_cont_tokens": "ecb050b141027f48"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0": {
|
|
"hashes": {
|
|
"hash_examples": "f2818986ee796bdd",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "83f120a531fb3e2a",
|
|
"hash_cont_tokens": "774146b60df6da60"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_medicine|0": {
|
|
"hashes": {
|
|
"hash_examples": "bfbac8bb6ff2adc7",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "3b4c1c73f59b5469",
|
|
"hash_cont_tokens": "6b97ef9c9d2810f6"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_medicine|0": {
|
|
"hashes": {
|
|
"hash_examples": "bfbac8bb6ff2adc7",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "1189b08a42e2f634",
|
|
"hash_cont_tokens": "633598661bd73f11"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0": {
|
|
"hashes": {
|
|
"hash_examples": "7c4135abff305666",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "50d3dc7196dfff51",
|
|
"hash_cont_tokens": "316ce90d2f2d0e0f"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"sklep.prompt2_nli_binary_sk:rte|0": {
|
|
"hashes": {
|
|
"hash_examples": "8ea06a4d7027e078",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "c0e09fc5609f1af2",
|
|
"hash_cont_tokens": "8279eee4a64f6547"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0": {
|
|
"hashes": {
|
|
"hash_examples": "2e0d5171d0d40ca2",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "057a4329cdd6e58d",
|
|
"hash_cont_tokens": "46f0e24c2c587524"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"sklep.prompt2_hate_speech_sk:hate-speech|0": {
|
|
"hashes": {
|
|
"hash_examples": "b3cd4f1ab55c772f",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "7505cc51168e73f6",
|
|
"hash_cont_tokens": "8728167288d94f99"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0": {
|
|
"hashes": {
|
|
"hash_examples": "7f3582a44ea04a78",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "702dd3167af66b42",
|
|
"hash_cont_tokens": "2600d71645cb0a4f"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0": {
|
|
"hashes": {
|
|
"hash_examples": "7c4135abff305666",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "a37f1fee9ddb74c7",
|
|
"hash_cont_tokens": "6c4429127f00cd1d"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0": {
|
|
"hashes": {
|
|
"hash_examples": "648884be05478291",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "cad3680d064b5933",
|
|
"hash_cont_tokens": "7365f46c693425df"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:marketing|0": {
|
|
"hashes": {
|
|
"hash_examples": "c4bcef0587679739",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "7e6350cc032822d9",
|
|
"hash_cont_tokens": "415bf899898a2a8c"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0": {
|
|
"hashes": {
|
|
"hash_examples": "8d92d327d53b2933",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "248010b481a96ce3",
|
|
"hash_cont_tokens": "5e3aaed86141a76b"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0": {
|
|
"hashes": {
|
|
"hash_examples": "7af87a7ae1cb0e53",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "d412e82c777fb52f",
|
|
"hash_cont_tokens": "cf2aaf8aff7b14cc"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:virology|0": {
|
|
"hashes": {
|
|
"hash_examples": "7c5d14ef48d0e860",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "a9e052f5318d6b5e",
|
|
"hash_cont_tokens": "3d5faee2fb339706"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0": {
|
|
"hashes": {
|
|
"hash_examples": "44459a57c28f5620",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "28139582f9049b37",
|
|
"hash_cont_tokens": "1d9de2bbe333fc6a"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0": {
|
|
"hashes": {
|
|
"hash_examples": "f6a1eb6fc83b4ee7",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "c7778a40d8d53049",
|
|
"hash_cont_tokens": "99db1895550f14b4"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0": {
|
|
"hashes": {
|
|
"hash_examples": "43cdbac391d8a1a9",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "5c800146e8f560f9",
|
|
"hash_cont_tokens": "0a3cf0d638dc970f"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:world_religions|0": {
|
|
"hashes": {
|
|
"hash_examples": "c1bbf26f068bb598",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "61a75455c51f4ae7",
|
|
"hash_cont_tokens": "63f3433623e54798"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"senti_sk.prompt3_sentiment_binary_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "33e538782d5b0e84",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "13c20804bf62b2ed",
|
|
"hash_cont_tokens": "4f050f3d04b35265"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0": {
|
|
"hashes": {
|
|
"hash_examples": "992e1937e8f9557c",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "ea4a28c1ab9215f5",
|
|
"hash_cont_tokens": "22ae6613b401fc8f"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"sklep.prompt1_nli_3_classes_sk:nli|0": {
|
|
"hashes": {
|
|
"hash_examples": "4ba8c3712f2beb9b",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "f7ab195fbc9b5768",
|
|
"hash_cont_tokens": "fa7cc85c20cc1a10"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0": {
|
|
"hashes": {
|
|
"hash_examples": "1cb2a8b29830cfad",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "b98540648f767a25",
|
|
"hash_cont_tokens": "99f2f20435299486"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0": {
|
|
"hashes": {
|
|
"hash_examples": "f6a1eb6fc83b4ee7",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "8c033a4d5267d677",
|
|
"hash_cont_tokens": "5a8db146b973ff63"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0": {
|
|
"hashes": {
|
|
"hash_examples": "ea7b1315dcf37a69",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "25a0f18af8b6f2a0",
|
|
"hash_cont_tokens": "d51a34da6c1fbf8b"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0": {
|
|
"hashes": {
|
|
"hash_examples": "c5e5e17570e2bb8c",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "c5949296a80706d7",
|
|
"hash_cont_tokens": "c0f6ff91bafd7713"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:business_ethics|0": {
|
|
"hashes": {
|
|
"hash_examples": "f891ffad3913452c",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "05003e5e7e31aed0",
|
|
"hash_cont_tokens": "1ece07f9dc1f9c22"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:public_relations|0": {
|
|
"hashes": {
|
|
"hash_examples": "74192a1a6507cb74",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "aeca9e8ab837f186",
|
|
"hash_cont_tokens": "b5fed06ce9a0c457"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0": {
|
|
"hashes": {
|
|
"hash_examples": "00f9ec3430780b27",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "bc7b7df7d3928228",
|
|
"hash_cont_tokens": "c418eef43e5ff47f"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"senti_sk.prompt2_sentiment_binary_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "48967df94707805e",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "0b0ee4739e0aa3ad",
|
|
"hash_cont_tokens": "9c1b73a32b734550"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0": {
|
|
"hashes": {
|
|
"hash_examples": "44459a57c28f5620",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "bac621d5b19b49c9",
|
|
"hash_cont_tokens": "f0de293fd5c86f70"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"belebele.prompt2_mqa_single_correct_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "40ce6eca2db6a52f",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "e1345cfe6c247089",
|
|
"hash_cont_tokens": "84169ff79e214a23"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:virology|0": {
|
|
"hashes": {
|
|
"hash_examples": "7c5d14ef48d0e860",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "56615f3e5db47185",
|
|
"hash_cont_tokens": "a946306d5cb5bd4b"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"sklep.prompt2_nli_3_classes_sk:nli|0": {
|
|
"hashes": {
|
|
"hash_examples": "4ba8c3712f2beb9b",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "23797da611879f2b",
|
|
"hash_cont_tokens": "3d355a5136418900"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0": {
|
|
"hashes": {
|
|
"hash_examples": "3fc0b7fe5c48a36c",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "d4dfebb59911599c",
|
|
"hash_cont_tokens": "be2b3240658584b3"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0": {
|
|
"hashes": {
|
|
"hash_examples": "1cb2a8b29830cfad",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "252942e119de6012",
|
|
"hash_cont_tokens": "02fcb3d20e2d1fad"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0": {
|
|
"hashes": {
|
|
"hash_examples": "c50a05f1347547cb",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "7fc690f1285f2fe0",
|
|
"hash_cont_tokens": "2da5682f64b00797"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0": {
|
|
"hashes": {
|
|
"hash_examples": "f66b2c8f1db1d216",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "29612ddcd200d9dd",
|
|
"hash_cont_tokens": "cf0f2bbdb1fa7243"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0": {
|
|
"hashes": {
|
|
"hash_examples": "7af87a7ae1cb0e53",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "44c4c682673a0e5a",
|
|
"hash_cont_tokens": "a03360803aaa6717"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0": {
|
|
"hashes": {
|
|
"hash_examples": "f84155f1bdc02fca",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "75941e07951d94b0",
|
|
"hash_cont_tokens": "7f07bc5c0339f6bd"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0": {
|
|
"hashes": {
|
|
"hash_examples": "b19a89d99d848605",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "7dff559e3f16a8d9",
|
|
"hash_cont_tokens": "8a0ef54196a324dc"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:security_studies|0": {
|
|
"hashes": {
|
|
"hash_examples": "9d7ec8fc0001ca04",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "9451192bda900677",
|
|
"hash_cont_tokens": "6abf5a54c7a1ba7e"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"skj1.prompt1_skj1_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "b288f09270a11fde",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "bf070bfbf9410180",
|
|
"hash_cont_tokens": "58372b1d0c345e75"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_physics|0": {
|
|
"hashes": {
|
|
"hash_examples": "c39cb2557a9b3972",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "16560bd9eded68a4",
|
|
"hash_cont_tokens": "d24d3889990f62cb"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"sklep.prompt3_hate_speech_sk:hate-speech|0": {
|
|
"hashes": {
|
|
"hash_examples": "b3cd4f1ab55c772f",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "64454b3f71edc8f0",
|
|
"hash_cont_tokens": "22e05c7d79fbf2b1"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0": {
|
|
"hashes": {
|
|
"hash_examples": "c50a05f1347547cb",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "a36778b8a176251b",
|
|
"hash_cont_tokens": "83688fdf12bf6e6a"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:virology|0": {
|
|
"hashes": {
|
|
"hash_examples": "7c5d14ef48d0e860",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "88f2fe5f90545f4c",
|
|
"hash_cont_tokens": "48606f764429804b"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0": {
|
|
"hashes": {
|
|
"hash_examples": "c50a05f1347547cb",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "ea6f8141161eff9c",
|
|
"hash_cont_tokens": "03a7cf0dde79ab06"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:security_studies|0": {
|
|
"hashes": {
|
|
"hash_examples": "9d7ec8fc0001ca04",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "c00402398e7df06b",
|
|
"hash_cont_tokens": "e66ce506f9da2a91"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"financial_exams.prompt3_completion_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "298560bc2b080ed4",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "9ccbde761f2b4c99",
|
|
"hash_cont_tokens": "afb6d8f336cee813"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0": {
|
|
"hashes": {
|
|
"hash_examples": "a3ec8c0cb9af6b38",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "65b266e8828d2912",
|
|
"hash_cont_tokens": "dc51ef05c787085a"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"senti_sk.prompt3_sentiment_3_class_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "77be771c44112b50",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "f774bb102b4544cf",
|
|
"hash_cont_tokens": "09d400de44c1ae08"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0": {
|
|
"hashes": {
|
|
"hash_examples": "dc8d9d90c47f46e4",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "0af57f4163626315",
|
|
"hash_cont_tokens": "2bb2bb33f8d2ea57"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"sklep.prompt3_qa_sk:question-answering|0": {
|
|
"hashes": {
|
|
"hash_examples": "387d6087cfc7dd8f",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "b267f5f74fcd520b",
|
|
"hash_cont_tokens": "bb89dad0f13100d5"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0": {
|
|
"hashes": {
|
|
"hash_examples": "5ca55eb406065df7",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "5702376dbd94a73e",
|
|
"hash_cont_tokens": "c397ae166819e71f"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:global_facts|0": {
|
|
"hashes": {
|
|
"hash_examples": "b92b2bc6891355a1",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "6f477d42372eb6a2",
|
|
"hash_cont_tokens": "84414ca9c05007ff"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:computer_security|0": {
|
|
"hashes": {
|
|
"hash_examples": "190389c0bf2799c1",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "64a28c1d4ccfc4c8",
|
|
"hash_cont_tokens": "e87564e25b1aad81"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:computer_security|0": {
|
|
"hashes": {
|
|
"hash_examples": "190389c0bf2799c1",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "9f6bc51ae575825f",
|
|
"hash_cont_tokens": "0c1f857e6db1e56e"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_physics|0": {
|
|
"hashes": {
|
|
"hash_examples": "c39cb2557a9b3972",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "a9331ecdd1e4830e",
|
|
"hash_cont_tokens": "3214166aa5eb6d7f"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"globalmmlu-lite.prompt2_mqa_single_correct_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "40308c6801250223",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "24091d8752abbeaa",
|
|
"hash_cont_tokens": "ffea3262e8a4d687"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0": {
|
|
"hashes": {
|
|
"hash_examples": "2c57bfc497c8083c",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "b211d0815455fdd5",
|
|
"hash_cont_tokens": "1cb2bcd292fe4c01"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:anatomy|0": {
|
|
"hashes": {
|
|
"hash_examples": "48e520a5e6a35fd8",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "9b167160771c1d25",
|
|
"hash_cont_tokens": "41051fc8916208dd"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:security_studies|0": {
|
|
"hashes": {
|
|
"hash_examples": "9d7ec8fc0001ca04",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "ac2cb98935cc711b",
|
|
"hash_cont_tokens": "5fc5c4ff5d04489f"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"movie_review.prompt3_sentiment_binary_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "cce02bac83d785bd",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "4965d78840c8cfd9",
|
|
"hash_cont_tokens": "95529fe2cc8512eb"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0": {
|
|
"hashes": {
|
|
"hash_examples": "fc35facc4060af81",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "b81bff7429195938",
|
|
"hash_cont_tokens": "45c38e83d9571bb5"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:astronomy|0": {
|
|
"hashes": {
|
|
"hash_examples": "ee6f5cfefae516d1",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "f33195ba9a67a08f",
|
|
"hash_cont_tokens": "64a5122b714b43a4"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0": {
|
|
"hashes": {
|
|
"hash_examples": "44459a57c28f5620",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "d60d3344ad76e89e",
|
|
"hash_cont_tokens": "6f9c1b83814ac536"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0": {
|
|
"hashes": {
|
|
"hash_examples": "1b29eb003b1f4bae",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "85ec8d09d026c1b4",
|
|
"hash_cont_tokens": "9284bccbf99b5c4f"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0": {
|
|
"hashes": {
|
|
"hash_examples": "dc8d9d90c47f46e4",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "9caa457ae591ab77",
|
|
"hash_cont_tokens": "0b49a2fe92118089"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0": {
|
|
"hashes": {
|
|
"hash_examples": "2c57bfc497c8083c",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "e0a45bbe0d3a0937",
|
|
"hash_cont_tokens": "0f606616138bab99"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:philosophy|0": {
|
|
"hashes": {
|
|
"hash_examples": "2265c96c25374e0d",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "12ae1a98f4417384",
|
|
"hash_cont_tokens": "43fe13827cf53aad"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0": {
|
|
"hashes": {
|
|
"hash_examples": "7f3582a44ea04a78",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "ba4b875526c4cf46",
|
|
"hash_cont_tokens": "a5811fbb8878fe23"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0": {
|
|
"hashes": {
|
|
"hash_examples": "1b29eb003b1f4bae",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "b19a194aa1d50e97",
|
|
"hash_cont_tokens": "1a7a59b45b0e8be9"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:world_religions|0": {
|
|
"hashes": {
|
|
"hash_examples": "c1bbf26f068bb598",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "f37bc31a030baa9a",
|
|
"hash_cont_tokens": "5b86a700b4becc79"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0": {
|
|
"hashes": {
|
|
"hash_examples": "f7fbee8f9cf664d7",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "bcab0cefe636430a",
|
|
"hash_cont_tokens": "3c8424cf270f9f3e"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"toxic_sk.prompt1_toxicity_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "d01cdad28edb6cc2",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "f0a31ed385787863",
|
|
"hash_cont_tokens": "cca90921d1085e3c"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"senti_sk.prompt1_sentiment_3_class_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "b190fde299643895",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "dd2cf0eb910c26af",
|
|
"hash_cont_tokens": "2773fbdc395f658f"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_biology|0": {
|
|
"hashes": {
|
|
"hash_examples": "8bd34873c0ac9972",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "01dcdfda57a14c51",
|
|
"hash_cont_tokens": "e698a72c294db6d5"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:public_relations|0": {
|
|
"hashes": {
|
|
"hash_examples": "74192a1a6507cb74",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "8a3e9186ecfb12ec",
|
|
"hash_cont_tokens": "54008dff8e859852"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0": {
|
|
"hashes": {
|
|
"hash_examples": "f7fbee8f9cf664d7",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "713a7980939a0201",
|
|
"hash_cont_tokens": "e3095155cb624465"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:formal_logic|0": {
|
|
"hashes": {
|
|
"hash_examples": "89e12f421bdc146a",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "1bde99bb98c7b811",
|
|
"hash_cont_tokens": "b89e4c9f6d5feb16"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0": {
|
|
"hashes": {
|
|
"hash_examples": "e2f6028f88e56ded",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "75b930014b12d245",
|
|
"hash_cont_tokens": "c1c8f4b92381d0c7"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:prehistory|0": {
|
|
"hashes": {
|
|
"hash_examples": "375708e013c71363",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "15fb08f445126658",
|
|
"hash_cont_tokens": "e865dbeccfdc7c97"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:econometrics|0": {
|
|
"hashes": {
|
|
"hash_examples": "7d90b1f77d986bbe",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "ded5b55309a02fc7",
|
|
"hash_cont_tokens": "3c6714f49aacb296"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"arc.prompt1_mqa_single_correct_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "6725e9266ede5f20",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "41b53dec6ce643a8",
|
|
"hash_cont_tokens": "28fe7acec29b9fb9"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:business_ethics|0": {
|
|
"hashes": {
|
|
"hash_examples": "f891ffad3913452c",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "6dd09ff1c6d50bcc",
|
|
"hash_cont_tokens": "9e46ec69a1eaf930"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0": {
|
|
"hashes": {
|
|
"hash_examples": "a3ec8c0cb9af6b38",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "d05b196a041fc49c",
|
|
"hash_cont_tokens": "47a93376504fa4e3"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"truthfulqa.prompt2_mqa_single_correct_sk:mc1|0": {
|
|
"hashes": {
|
|
"hash_examples": "ee751399d6f4ae5f",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "347d8c24b9b0fa76",
|
|
"hash_cont_tokens": "67eb7937d8d6ede4"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0": {
|
|
"hashes": {
|
|
"hash_examples": "81792091d0a281a1",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "1446b11a62794dd5",
|
|
"hash_cont_tokens": "d1658cdc2bbd7c21"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:astronomy|0": {
|
|
"hashes": {
|
|
"hash_examples": "ee6f5cfefae516d1",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "688bd5d3ae4f3327",
|
|
"hash_cont_tokens": "52f4493b7f987643"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0": {
|
|
"hashes": {
|
|
"hash_examples": "3fc0b7fe5c48a36c",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "f12179ec42cdbd8b",
|
|
"hash_cont_tokens": "5db3a0c85c74416c"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"senti_sk.prompt2_sentiment_3_class_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "e15f449b9f949296",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "31b44dbb87a1e410",
|
|
"hash_cont_tokens": "ba0197d7d9005547"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0": {
|
|
"hashes": {
|
|
"hash_examples": "3fc0b7fe5c48a36c",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "680e80f0187d40b8",
|
|
"hash_cont_tokens": "4d1786164b96bed1"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:formal_logic|0": {
|
|
"hashes": {
|
|
"hash_examples": "89e12f421bdc146a",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "2647ed914277ad4a",
|
|
"hash_cont_tokens": "440826ea5784b5cd"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0": {
|
|
"hashes": {
|
|
"hash_examples": "c5e5e17570e2bb8c",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "514875b657d2a2e7",
|
|
"hash_cont_tokens": "38f535b0d5304e6b"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0": {
|
|
"hashes": {
|
|
"hash_examples": "4f05c737d92acb1b",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "3d293b6cc7ba3944",
|
|
"hash_cont_tokens": "a5674a7e851f8650"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0": {
|
|
"hashes": {
|
|
"hash_examples": "448c6360a63e82fc",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "52bdfcdaefdae0a8",
|
|
"hash_cont_tokens": "d5bb4872feed277f"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"skj1.prompt2_skj1_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "b288f09270a11fde",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "3be1728f5c52d964",
|
|
"hash_cont_tokens": "721ed1cd853c6e69"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:international_law|0": {
|
|
"hashes": {
|
|
"hash_examples": "a3aec6cd58af642c",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "70f3343aa62cfcc7",
|
|
"hash_cont_tokens": "2dd13507c763ffa5"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"sklep.prompt1_nli_binary_sk:rte|0": {
|
|
"hashes": {
|
|
"hash_examples": "8ea06a4d7027e078",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "e0e9316ffd581271",
|
|
"hash_cont_tokens": "be0550ba11635e11"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:human_aging|0": {
|
|
"hashes": {
|
|
"hash_examples": "1d0dd017926beb00",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "4f05f6e7a550c21a",
|
|
"hash_cont_tokens": "54458199d9af0a5e"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:management|0": {
|
|
"hashes": {
|
|
"hash_examples": "6be6db70af55c301",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "62299f6e7e6dd7ad",
|
|
"hash_cont_tokens": "0b996d5a7366c705"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0": {
|
|
"hashes": {
|
|
"hash_examples": "ce1327e99680eaa4",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "262eefd819afdf80",
|
|
"hash_cont_tokens": "6da4120628b9ad37"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"truthfulqa.prompt1_mqa_single_correct_sk:mc1|0": {
|
|
"hashes": {
|
|
"hash_examples": "ee751399d6f4ae5f",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "8fb4041d9b70121f",
|
|
"hash_cont_tokens": "caabb41ccba0a6ac"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"senti_sk.prompt1_sentiment_binary_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "394f7fff1d43e549",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "d3a66d1e658ad28e",
|
|
"hash_cont_tokens": "7e88c9420eb5058d"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:astronomy|0": {
|
|
"hashes": {
|
|
"hash_examples": "ee6f5cfefae516d1",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "d0ad75a60f59cdf4",
|
|
"hash_cont_tokens": "6986841198743675"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:human_aging|0": {
|
|
"hashes": {
|
|
"hash_examples": "1d0dd017926beb00",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "a93f9a4c95f94c4e",
|
|
"hash_cont_tokens": "fab496ef83c3edef"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:formal_logic|0": {
|
|
"hashes": {
|
|
"hash_examples": "89e12f421bdc146a",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "ffe03d00391d95e2",
|
|
"hash_cont_tokens": "b935ca66658fcebb"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:sociology|0": {
|
|
"hashes": {
|
|
"hash_examples": "dff8fca926f7b793",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "b2f3eaea6d328989",
|
|
"hash_cont_tokens": "8136d16dda96abd7"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:econometrics|0": {
|
|
"hashes": {
|
|
"hash_examples": "7d90b1f77d986bbe",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "df280927b76e86c3",
|
|
"hash_cont_tokens": "d7f0770f9a0acc43"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0": {
|
|
"hashes": {
|
|
"hash_examples": "ac8d3b0039dcb239",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "348619f08a3cb49f",
|
|
"hash_cont_tokens": "2587b22c786990ab"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0": {
|
|
"hashes": {
|
|
"hash_examples": "5bc6e3e3c9ba587f",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "de95e8a190e7106e",
|
|
"hash_cont_tokens": "ab9a76e59c0de440"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0": {
|
|
"hashes": {
|
|
"hash_examples": "5ca55eb406065df7",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "af10b44637f3d938",
|
|
"hash_cont_tokens": "ba9094d4854ad209"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0": {
|
|
"hashes": {
|
|
"hash_examples": "5bc6e3e3c9ba587f",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "3f9133a48bea2708",
|
|
"hash_cont_tokens": "99d4ea17fc3b67b1"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"globalmmlu-lite.prompt3_mqa_single_correct_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "40308c6801250223",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "f048bcc8c8407fab",
|
|
"hash_cont_tokens": "5eb598a9cf4b1a43"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0": {
|
|
"hashes": {
|
|
"hash_examples": "1cb2a8b29830cfad",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "cc534b0b484b0889",
|
|
"hash_cont_tokens": "895b0ada8425f24e"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0": {
|
|
"hashes": {
|
|
"hash_examples": "2e0d5171d0d40ca2",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "4c0430a1d48a772f",
|
|
"hash_cont_tokens": "118cc18af5afd220"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0": {
|
|
"hashes": {
|
|
"hash_examples": "7f3582a44ea04a78",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "a3caec54057832f7",
|
|
"hash_cont_tokens": "7a2ebf934c6ddd47"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:computer_security|0": {
|
|
"hashes": {
|
|
"hash_examples": "190389c0bf2799c1",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "29e630b4667d806e",
|
|
"hash_cont_tokens": "fc8bb5f88f776ddd"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"toxic_sk.prompt2_toxicity_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "d01cdad28edb6cc2",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "7f740765062a487f",
|
|
"hash_cont_tokens": "bace2bcdcd43efff"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"multi_wiki_qa.prompt3_qa_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "ee9acf278d86ea6b",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "a691374d2eecdcc1",
|
|
"hash_cont_tokens": "613fd2b1ac9a5cfd"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"financial_exams.prompt1_completion_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "298560bc2b080ed4",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "2b48a01eefc2cb5a",
|
|
"hash_cont_tokens": "5e32bbcae590cd3d"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:philosophy|0": {
|
|
"hashes": {
|
|
"hash_examples": "2265c96c25374e0d",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "c529c877230b655e",
|
|
"hash_cont_tokens": "224aeb6b81f7af4a"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0": {
|
|
"hashes": {
|
|
"hash_examples": "b19a89d99d848605",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "9cca5de891f88280",
|
|
"hash_cont_tokens": "46863e3a251ea603"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:marketing|0": {
|
|
"hashes": {
|
|
"hash_examples": "c4bcef0587679739",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "a61868461bf00ce3",
|
|
"hash_cont_tokens": "a975feda43d08a31"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"globalmmlu-lite.prompt1_mqa_single_correct_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "40308c6801250223",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "904359bdb1a2a0c5",
|
|
"hash_cont_tokens": "91569cbd7a98e2e7"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_law|0": {
|
|
"hashes": {
|
|
"hash_examples": "645c5fc320444ab9",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "fecb378080561a4e",
|
|
"hash_cont_tokens": "7efe7abd05372091"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:machine_learning|0": {
|
|
"hashes": {
|
|
"hash_examples": "9c6bb044db076a06",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "695fbc65bb4b4749",
|
|
"hash_cont_tokens": "1773964b488ee162"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0": {
|
|
"hashes": {
|
|
"hash_examples": "43cdbac391d8a1a9",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "a53ed6b9166977dd",
|
|
"hash_cont_tokens": "811e17a82722f4b1"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0": {
|
|
"hashes": {
|
|
"hash_examples": "c5e5e17570e2bb8c",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "7e294dff86f1a2d3",
|
|
"hash_cont_tokens": "d5c498abba098875"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0": {
|
|
"hashes": {
|
|
"hash_examples": "992e1937e8f9557c",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "205358a7cc994b13",
|
|
"hash_cont_tokens": "0e06e5533f04bd97"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:management|0": {
|
|
"hashes": {
|
|
"hash_examples": "6be6db70af55c301",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "c254897027f8e08a",
|
|
"hash_cont_tokens": "e957e86212786d95"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:sociology|0": {
|
|
"hashes": {
|
|
"hash_examples": "dff8fca926f7b793",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "db5ae2473d1f67fc",
|
|
"hash_cont_tokens": "4195ee7cc86517a2"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0": {
|
|
"hashes": {
|
|
"hash_examples": "43cdbac391d8a1a9",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "cfcede6c298395d7",
|
|
"hash_cont_tokens": "256e6972f4cf49d5"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"skj1.prompt3_skj1_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "b288f09270a11fde",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "8e25eaf65ffc8542",
|
|
"hash_cont_tokens": "fdaf8c910880d507"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"movie_review.prompt1_sentiment_binary_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "ec65ab0c1f9d54f6",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "6f783ecc2d68c0d4",
|
|
"hash_cont_tokens": "eaac3ce2d9682b3c"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:anatomy|0": {
|
|
"hashes": {
|
|
"hash_examples": "48e520a5e6a35fd8",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "df796a450e66859d",
|
|
"hash_cont_tokens": "70ee668bfc0bf1a1"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0": {
|
|
"hashes": {
|
|
"hash_examples": "f84155f1bdc02fca",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "7aabc5016dc3937e",
|
|
"hash_cont_tokens": "52cde7c1b5f52f44"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:philosophy|0": {
|
|
"hashes": {
|
|
"hash_examples": "2265c96c25374e0d",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "7dad55635ca272a2",
|
|
"hash_cont_tokens": "a0978611d159be0b"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:human_aging|0": {
|
|
"hashes": {
|
|
"hash_examples": "1d0dd017926beb00",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "0cdb4ea308771c51",
|
|
"hash_cont_tokens": "c3223f47a2379678"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0": {
|
|
"hashes": {
|
|
"hash_examples": "ac8d3b0039dcb239",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "09ef2052a952a1f4",
|
|
"hash_cont_tokens": "efb1518f8c99d455"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0": {
|
|
"hashes": {
|
|
"hash_examples": "f84155f1bdc02fca",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "ec80ae616fd5c56b",
|
|
"hash_cont_tokens": "1f31bbdf0bd4bc9f"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0": {
|
|
"hashes": {
|
|
"hash_examples": "ea73340f4d960eee",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "8a7945fdabc42eb4",
|
|
"hash_cont_tokens": "07e1a1a849acff1c"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0": {
|
|
"hashes": {
|
|
"hash_examples": "5ca55eb406065df7",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "00c9648dbca90eb4",
|
|
"hash_cont_tokens": "83b5352b627600c0"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0": {
|
|
"hashes": {
|
|
"hash_examples": "dc8d9d90c47f46e4",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "a83a255e12ee5c30",
|
|
"hash_cont_tokens": "db8f53776e03132c"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0": {
|
|
"hashes": {
|
|
"hash_examples": "8d92d327d53b2933",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "9c06353f63194270",
|
|
"hash_cont_tokens": "93f4268a9a43e51b"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"sklep.prompt1_hate_speech_sk:hate-speech|0": {
|
|
"hashes": {
|
|
"hash_examples": "75602e70ff2fabe7",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "57dde360cdf42787",
|
|
"hash_cont_tokens": "d145a63fa10283aa"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0": {
|
|
"hashes": {
|
|
"hash_examples": "81792091d0a281a1",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "6d03015e20d48f39",
|
|
"hash_cont_tokens": "f3c8d6a4f1f41790"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:world_religions|0": {
|
|
"hashes": {
|
|
"hash_examples": "c1bbf26f068bb598",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "ac5577442c9ac24e",
|
|
"hash_cont_tokens": "d839a2024ab476f9"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0": {
|
|
"hashes": {
|
|
"hash_examples": "27b6684dc6c0d1b3",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "d88fa2f69f58c729",
|
|
"hash_cont_tokens": "6057522afb931155"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0": {
|
|
"hashes": {
|
|
"hash_examples": "2c57bfc497c8083c",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "92997f9687899a24",
|
|
"hash_cont_tokens": "213d63dc03cb0502"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_biology|0": {
|
|
"hashes": {
|
|
"hash_examples": "8bd34873c0ac9972",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "a8f972b53e98ef2a",
|
|
"hash_cont_tokens": "64e41340c060fb2c"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:global_facts|0": {
|
|
"hashes": {
|
|
"hash_examples": "b92b2bc6891355a1",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "2c2346af6175d9a6",
|
|
"hash_cont_tokens": "02b1ec2d4956227f"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0": {
|
|
"hashes": {
|
|
"hash_examples": "b19a89d99d848605",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "011c1542b14a0f48",
|
|
"hash_cont_tokens": "0dd254aecec04233"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0": {
|
|
"hashes": {
|
|
"hash_examples": "e2f6028f88e56ded",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "0694ec2172ab5650",
|
|
"hash_cont_tokens": "c660ab315ca526ef"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0": {
|
|
"hashes": {
|
|
"hash_examples": "f6a1eb6fc83b4ee7",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "bd2bf17bbe995b0a",
|
|
"hash_cont_tokens": "89e953ec48e709f3"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_physics|0": {
|
|
"hashes": {
|
|
"hash_examples": "c39cb2557a9b3972",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "6e8c68e90ea666f3",
|
|
"hash_cont_tokens": "77498a00a20ba9ef"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0": {
|
|
"hashes": {
|
|
"hash_examples": "ea73340f4d960eee",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "6959bb25755ad88e",
|
|
"hash_cont_tokens": "0e6b7a05add605a8"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0": {
|
|
"hashes": {
|
|
"hash_examples": "8d92d327d53b2933",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "e18b35af08465d62",
|
|
"hash_cont_tokens": "3754583dd7de6bb2"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"sklep.prompt3_nli_binary_sk:rte|0": {
|
|
"hashes": {
|
|
"hash_examples": "8ea06a4d7027e078",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "9b7b2b80c583cf75",
|
|
"hash_cont_tokens": "67142a1c5de16574"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"slovakcopa.prompt2_copa_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "3adee78dd6203724",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "248b9d8a04f8c062",
|
|
"hash_cont_tokens": "20ad2370f3961274"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0": {
|
|
"hashes": {
|
|
"hash_examples": "a3ec8c0cb9af6b38",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "fbfa3f6159a2611c",
|
|
"hash_cont_tokens": "a521f941e5eee9df"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"multi_wiki_qa.prompt1_qa_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "f166d5d501f82d9e",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "a787e89b7b7e6958",
|
|
"hash_cont_tokens": "f9d012e1e6a49349"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0": {
|
|
"hashes": {
|
|
"hash_examples": "7c4135abff305666",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "80e30ebae0872849",
|
|
"hash_cont_tokens": "4f427c2cde109462"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:global_facts|0": {
|
|
"hashes": {
|
|
"hash_examples": "b92b2bc6891355a1",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "39c0d368486ca8f0",
|
|
"hash_cont_tokens": "67f5a60662be2e7d"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:sociology|0": {
|
|
"hashes": {
|
|
"hash_examples": "dff8fca926f7b793",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "0966718588555e05",
|
|
"hash_cont_tokens": "35435678522c4939"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0": {
|
|
"hashes": {
|
|
"hash_examples": "27b6684dc6c0d1b3",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "abd87e6ae42adf81",
|
|
"hash_cont_tokens": "e3ef820b35709164"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:public_relations|0": {
|
|
"hashes": {
|
|
"hash_examples": "74192a1a6507cb74",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "2222531cc6e9cd92",
|
|
"hash_cont_tokens": "c430b74fedea99c2"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0": {
|
|
"hashes": {
|
|
"hash_examples": "00f9ec3430780b27",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "fb4be485e3061b23",
|
|
"hash_cont_tokens": "4d687025986f0143"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:anatomy|0": {
|
|
"hashes": {
|
|
"hash_examples": "48e520a5e6a35fd8",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "cee5c9ddd78fd571",
|
|
"hash_cont_tokens": "02da6630b897eeb5"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:marketing|0": {
|
|
"hashes": {
|
|
"hash_examples": "c4bcef0587679739",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "2b8aa9ed12f6b266",
|
|
"hash_cont_tokens": "e7a7c554c130b919"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0": {
|
|
"hashes": {
|
|
"hash_examples": "ce1327e99680eaa4",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "ee5e80f1825f089c",
|
|
"hash_cont_tokens": "51ca5d6d2b8ca9ad"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0": {
|
|
"hashes": {
|
|
"hash_examples": "e2f6028f88e56ded",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "8283452d92579ef1",
|
|
"hash_cont_tokens": "27454841e4594359"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"slovakcopa.prompt1_copa_sk|0": {
|
|
"hashes": {
|
|
"hash_examples": "37b56578071ef6b2",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "8c6a5ae3cc5489ef",
|
|
"hash_cont_tokens": "47e39104d5b846c3"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:nutrition|0": {
|
|
"hashes": {
|
|
"hash_examples": "47563eff5f4089bf",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "751607ba52c4d615",
|
|
"hash_cont_tokens": "e750e09146977427"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:machine_learning|0": {
|
|
"hashes": {
|
|
"hash_examples": "9c6bb044db076a06",
|
|
"hash_full_prompts": "ef46db3751d8e999",
|
|
"hash_input_tokens": "8b56dca09a4cead4",
|
|
"hash_cont_tokens": "a8155bb958c58d4c"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
}
|
|
},
|
|
"summary_general": {
|
|
"hashes": {
|
|
"hash_examples": "316f54894dcb6f0d",
|
|
"hash_full_prompts": "ba7f9b257c923f55",
|
|
"hash_input_tokens": "8de2f010e9daf7fe",
|
|
"hash_cont_tokens": "1f06ad3e4b80f7cc"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
}
|
|
} |