3250 lines
105 KiB
JSON
3250 lines
105 KiB
JSON
{
|
|
"model_name": "models/ministral-base",
|
|
"config_general": {
|
|
"lighteval_sha": "?",
|
|
"num_fewshot_seeds": 1,
|
|
"max_samples": null,
|
|
"job_id": "0",
|
|
"start_time": 883355.287443326,
|
|
"end_time": 883520.97395618,
|
|
"total_evaluation_time_secondes": "165.68651285406668",
|
|
"model_config": {
|
|
"model_name": "models/ministral-base",
|
|
"generation_parameters": {
|
|
"num_blocks": null,
|
|
"block_size": null,
|
|
"early_stopping": null,
|
|
"repetition_penalty": null,
|
|
"frequency_penalty": null,
|
|
"length_penalty": null,
|
|
"presence_penalty": null,
|
|
"max_new_tokens": null,
|
|
"min_new_tokens": null,
|
|
"seed": null,
|
|
"stop_tokens": null,
|
|
"temperature": 0,
|
|
"top_k": null,
|
|
"min_p": null,
|
|
"top_p": null,
|
|
"truncate_prompt": null,
|
|
"cache_implementation": null,
|
|
"response_format": null,
|
|
"skip_special_tokens": true
|
|
},
|
|
"system_prompt": null,
|
|
"cache_dir": "~/.cache/huggingface/lighteval",
|
|
"tokenizer": null,
|
|
"revision": "main",
|
|
"dtype": "bfloat16",
|
|
"tensor_parallel_size": 1,
|
|
"data_parallel_size": 1,
|
|
"pipeline_parallel_size": 1,
|
|
"gpu_memory_utilization": 0.9,
|
|
"enable_prefix_caching": null,
|
|
"max_model_length": null,
|
|
"quantization": null,
|
|
"load_format": null,
|
|
"swap_space": 4,
|
|
"seed": 1234,
|
|
"trust_remote_code": true,
|
|
"add_special_tokens": true,
|
|
"multichoice_continuations_start_space": true,
|
|
"pairwise_tokenization": false,
|
|
"max_num_seqs": 128,
|
|
"max_num_batched_tokens": 2048,
|
|
"subfolder": null,
|
|
"is_async": false,
|
|
"override_chat_template": null,
|
|
"enforce_eager": false
|
|
},
|
|
"model_name": "models/ministral-base"
|
|
},
|
|
"versions": {},
|
|
"results": {
|
|
"e_tests.prompt1_mqa_single_correct_sk:slovensky-jazyk|0": {
|
|
"acc": 0.25109170305676853,
|
|
"acc_stderr": 0.020284878098207157,
|
|
"em_relaxed": 0.25327510917030566,
|
|
"em_relaxed_stderr": 0.020343162564319364
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0": {
|
|
"acc": 0.19638242894056848,
|
|
"acc_stderr": 0.020220056896337255,
|
|
"em_relaxed": 0.19896640826873385,
|
|
"em_relaxed_stderr": 0.02031990120797851
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:philosophy|0": {
|
|
"acc": 0.12337662337662338,
|
|
"acc_stderr": 0.01876954635295657,
|
|
"em_relaxed": 0.12337662337662338,
|
|
"em_relaxed_stderr": 0.01876954635295657
|
|
},
|
|
"scio.prompt1_mqa_single_correct_sk:mat|0": {
|
|
"acc": 0.23497267759562843,
|
|
"acc_stderr": 0.022192218701649325,
|
|
"em_relaxed": 0.2459016393442623,
|
|
"em_relaxed_stderr": 0.02253970761756048
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:security_studies|0": {
|
|
"acc": 0.1981981981981982,
|
|
"acc_stderr": 0.026815582053277193,
|
|
"em_relaxed": 0.20270270270270271,
|
|
"em_relaxed_stderr": 0.027042309486278813
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0": {
|
|
"acc": 0.20522388059701493,
|
|
"acc_stderr": 0.017460611985170203,
|
|
"em_relaxed": 0.21082089552238806,
|
|
"em_relaxed_stderr": 0.017634685741009386
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:security_studies|0": {
|
|
"acc": 0.16666666666666666,
|
|
"acc_stderr": 0.02506903489984115,
|
|
"em_relaxed": 0.1891891891891892,
|
|
"em_relaxed_stderr": 0.02634582401273633
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0": {
|
|
"acc": 0.1891891891891892,
|
|
"acc_stderr": 0.02438361375395457,
|
|
"em_relaxed": 0.1891891891891892,
|
|
"em_relaxed_stderr": 0.02438361375395457
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0": {
|
|
"acc": 0.2644927536231884,
|
|
"acc_stderr": 0.026597083702337265,
|
|
"em_relaxed": 0.26811594202898553,
|
|
"em_relaxed_stderr": 0.02671259720934512
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0": {
|
|
"acc": 0.2878787878787879,
|
|
"acc_stderr": 0.03225883512300992,
|
|
"em_relaxed": 0.2878787878787879,
|
|
"em_relaxed_stderr": 0.03225883512300992
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0": {
|
|
"acc": 0.17,
|
|
"acc_stderr": 0.03775251680686371,
|
|
"em_relaxed": 0.17,
|
|
"em_relaxed_stderr": 0.03775251680686371
|
|
},
|
|
"slovak_tests.prompt3_qa_sk:maturita|0": {
|
|
"qem": 0.01327433628318584,
|
|
"qem_stderr": 0.007629803632081149,
|
|
"qf1": 0.0022123893805309734,
|
|
"qf1_stderr": 0.0015609152277667225,
|
|
"rouge1": 0.0013282561753440866,
|
|
"rouge1_stderr": 0.0009107295550233445,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0013282561753440866,
|
|
"rougeL_stderr": 0.0009107295550233445
|
|
},
|
|
"slovak_tests.prompt1_mqa_single_correct_sk:monitor9|0": {
|
|
"acc": 0.24423963133640553,
|
|
"acc_stderr": 0.029232990766796058,
|
|
"em_relaxed": 0.24423963133640553,
|
|
"em_relaxed_stderr": 0.029232990766796058
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_medicine|0": {
|
|
"acc": 0.25,
|
|
"acc_stderr": 0.033113308926626096,
|
|
"em_relaxed": 0.25,
|
|
"em_relaxed_stderr": 0.033113308926626096
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0": {
|
|
"acc": 0.3939393939393939,
|
|
"acc_stderr": 0.04935824351078519,
|
|
"em_relaxed": 0.41414141414141414,
|
|
"em_relaxed_stderr": 0.04975740158605099
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:machine_learning|0": {
|
|
"acc": 0.16964285714285715,
|
|
"acc_stderr": 0.035623678500953895,
|
|
"em_relaxed": 0.1875,
|
|
"em_relaxed_stderr": 0.0370468111477387
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:virology|0": {
|
|
"acc": 0.17682926829268292,
|
|
"acc_stderr": 0.02988327785748596,
|
|
"em_relaxed": 0.18902439024390244,
|
|
"em_relaxed_stderr": 0.030666839281449514
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:anatomy|0": {
|
|
"acc": 0.2462686567164179,
|
|
"acc_stderr": 0.037358294208002164,
|
|
"em_relaxed": 0.2537313432835821,
|
|
"em_relaxed_stderr": 0.037731914327133735
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:global_facts|0": {
|
|
"acc": 0.29,
|
|
"acc_stderr": 0.045604802157206845,
|
|
"em_relaxed": 0.3,
|
|
"em_relaxed_stderr": 0.046056618647183814
|
|
},
|
|
"e_tests.prompt2_mqa_single_correct_sk:geografia|0": {
|
|
"acc": 0.2857142857142857,
|
|
"acc_stderr": 0.06520506636966263,
|
|
"em_relaxed": 0.2857142857142857,
|
|
"em_relaxed_stderr": 0.06520506636966263
|
|
},
|
|
"e_tests.prompt2_mqa_single_correct_sk:chemia|0": {
|
|
"acc": 0.07142857142857142,
|
|
"acc_stderr": 0.034726602486028414,
|
|
"em_relaxed": 0.08928571428571429,
|
|
"em_relaxed_stderr": 0.038450387280282494
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0": {
|
|
"acc": 0.1656441717791411,
|
|
"acc_stderr": 0.020621585415044594,
|
|
"em_relaxed": 0.17484662576687116,
|
|
"em_relaxed_stderr": 0.021069502803436863
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:econometrics|0": {
|
|
"acc": 0.1875,
|
|
"acc_stderr": 0.0370468111477387,
|
|
"em_relaxed": 0.19642857142857142,
|
|
"em_relaxed_stderr": 0.03770970049347017
|
|
},
|
|
"movie_review.prompt2_sentiment_binary_sk|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"e_tests.prompt1_mqa_single_correct_sk:biologia|0": {
|
|
"acc": 0.19540229885057472,
|
|
"acc_stderr": 0.0427567811097387,
|
|
"em_relaxed": 0.20689655172413793,
|
|
"em_relaxed_stderr": 0.04368097459950702
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:management|0": {
|
|
"acc": 0.21,
|
|
"acc_stderr": 0.040936018074033256,
|
|
"em_relaxed": 0.21,
|
|
"em_relaxed_stderr": 0.040936018074033256
|
|
},
|
|
"e_tests.prompt3_mqa_multiple_correct_sk:fyzika|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:formal_logic|0": {
|
|
"acc": 0.1565217391304348,
|
|
"acc_stderr": 0.034030776100725475,
|
|
"em_relaxed": 0.2,
|
|
"em_relaxed_stderr": 0.03746343246326772
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0": {
|
|
"acc": 0.22,
|
|
"acc_stderr": 0.04163331998932268,
|
|
"em_relaxed": 0.22,
|
|
"em_relaxed_stderr": 0.04163331998932268
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0": {
|
|
"acc": 0.22727272727272727,
|
|
"acc_stderr": 0.029857515673386393,
|
|
"em_relaxed": 0.23232323232323232,
|
|
"em_relaxed_stderr": 0.030088629490217483
|
|
},
|
|
"movie_review.prompt3_sentiment_binary_sk|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slovak_tests.prompt2_qa_sk:maturita|0": {
|
|
"qem": 0.01327433628318584,
|
|
"qem_stderr": 0.007629803632081149,
|
|
"qf1": 0.002753195673549656,
|
|
"qf1_stderr": 0.0020208822149325673,
|
|
"rouge1": 0.0008509189925119129,
|
|
"rouge1_stderr": 0.0007009516038640078,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0008509189925119129,
|
|
"rougeL_stderr": 0.0007009516038640078
|
|
},
|
|
"e_tests.prompt1_qa_sk:slovensky-jazyk|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0025906735751295338,
|
|
"qf1_stderr": 0.0025906735751295403,
|
|
"rouge1": 5.842773836305639e-05,
|
|
"rouge1_stderr": 4.122343590294019e-05,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 5.842773836305639e-05,
|
|
"rougeL_stderr": 4.122343590294019e-05
|
|
},
|
|
"e_tests.prompt2_qa_sk:dejepis|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"e_tests.prompt3_mqa_single_correct_sk:slovensky-jazyk|0": {
|
|
"acc": 0.2205240174672489,
|
|
"acc_stderr": 0.019394170839568875,
|
|
"em_relaxed": 0.22270742358078602,
|
|
"em_relaxed_stderr": 0.019462629121054217
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:astronomy|0": {
|
|
"acc": 0.23026315789473684,
|
|
"acc_stderr": 0.03426059424403165,
|
|
"em_relaxed": 0.23684210526315788,
|
|
"em_relaxed_stderr": 0.03459777606810535
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0": {
|
|
"acc": 0.2702702702702703,
|
|
"acc_stderr": 0.02764841916772191,
|
|
"em_relaxed": 0.28185328185328185,
|
|
"em_relaxed_stderr": 0.028009688894113486
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0": {
|
|
"acc": 0.21100917431192662,
|
|
"acc_stderr": 0.039262212380368526,
|
|
"em_relaxed": 0.22018348623853212,
|
|
"em_relaxed_stderr": 0.03987279758533731
|
|
},
|
|
"e_tests.prompt1_mqa_multiple_correct_sk:chemia|0": {
|
|
"acc": 0.045454545454545456,
|
|
"acc_stderr": 0.031765283723714
|
|
},
|
|
"slovak_tests.prompt3_mqa_single_correct_sk:monitor9|0": {
|
|
"acc": 0.19815668202764977,
|
|
"acc_stderr": 0.02712203315362448,
|
|
"em_relaxed": 0.19815668202764977,
|
|
"em_relaxed_stderr": 0.02712203315362448
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0": {
|
|
"acc": 0.2092426187419769,
|
|
"acc_stderr": 0.014583336673745648,
|
|
"em_relaxed": 0.21181001283697048,
|
|
"em_relaxed_stderr": 0.014648693814173547
|
|
},
|
|
"e_tests.prompt3_mqa_multiple_correct_sk:informatika|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"e_tests.prompt2_mqa_single_correct_sk:biologia|0": {
|
|
"acc": 0.14942528735632185,
|
|
"acc_stderr": 0.03844315810763212,
|
|
"em_relaxed": 0.14942528735632185,
|
|
"em_relaxed_stderr": 0.03844315810763212
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:virology|0": {
|
|
"acc": 0.23780487804878048,
|
|
"acc_stderr": 0.03334645408665337,
|
|
"em_relaxed": 0.24390243902439024,
|
|
"em_relaxed_stderr": 0.03363591048272823
|
|
},
|
|
"e_tests.prompt1_mqa_multiple_correct_sk:obcianska-nauka|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:human_aging|0": {
|
|
"acc": 0.1232876712328767,
|
|
"acc_stderr": 0.022266919281488138,
|
|
"em_relaxed": 0.1232876712328767,
|
|
"em_relaxed_stderr": 0.022266919281488138
|
|
},
|
|
"e_tests.prompt1_qa_sk:geografia|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"sklep.prompt3_nli_3_classes_sk:nli|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"scio.prompt3_mqa_single_correct_sk:vsp|0": {
|
|
"acc": 0.20388349514563106,
|
|
"acc_stderr": 0.014043631695334572,
|
|
"em_relaxed": 0.21723300970873785,
|
|
"em_relaxed_stderr": 0.014374053054620224
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0": {
|
|
"acc": 0.2848101265822785,
|
|
"acc_stderr": 0.03601957344556447,
|
|
"em_relaxed": 0.2848101265822785,
|
|
"em_relaxed_stderr": 0.03601957344556447
|
|
},
|
|
"sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"arc.prompt1_mqa_single_correct_sk|0": {
|
|
"acc": 0.2728828058169376,
|
|
"acc_stderr": 0.013033734552785697,
|
|
"em_relaxed": 0.2728828058169376,
|
|
"em_relaxed_stderr": 0.013033734552785697
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0": {
|
|
"acc": 0.18181818181818182,
|
|
"acc_stderr": 0.038961038961039,
|
|
"em_relaxed": 0.18181818181818182,
|
|
"em_relaxed_stderr": 0.038961038961039
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0": {
|
|
"acc": 0.2616822429906542,
|
|
"acc_stderr": 0.0426929191572811,
|
|
"em_relaxed": 0.27102803738317754,
|
|
"em_relaxed_stderr": 0.04317273776566666
|
|
},
|
|
"belebele.prompt2_mqa_single_correct_sk|0": {
|
|
"acc": 0.2511111111111111,
|
|
"acc_stderr": 0.014463114105170805,
|
|
"em_relaxed": 0.25666666666666665,
|
|
"em_relaxed_stderr": 0.01456789134238002
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_law|0": {
|
|
"acc": 0.20922288642186165,
|
|
"acc_stderr": 0.0118915566619952,
|
|
"em_relaxed": 0.21178479931682323,
|
|
"em_relaxed_stderr": 0.011944744511660903
|
|
},
|
|
"senti_sk.prompt2_sentiment_3_class_sk|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0": {
|
|
"acc": 0.22018348623853212,
|
|
"acc_stderr": 0.03987279758533728,
|
|
"em_relaxed": 0.22018348623853212,
|
|
"em_relaxed_stderr": 0.03987279758533728
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0": {
|
|
"acc": 0.13333333333333333,
|
|
"acc_stderr": 0.029365879728106854,
|
|
"em_relaxed": 0.13333333333333333,
|
|
"em_relaxed_stderr": 0.029365879728106854
|
|
},
|
|
"e_tests.prompt1_mqa_multiple_correct_sk:slovensky-jazyk|0": {
|
|
"acc": 0.014285714285714285,
|
|
"acc_stderr": 0.010065123433632259
|
|
},
|
|
"sklep.prompt2_hate_speech_sk:hate-speech|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slovakcopa.prompt3_copa_sk|0": {
|
|
"acc": 0.144,
|
|
"acc_stderr": 0.015716934945725767,
|
|
"em_relaxed": 0.144,
|
|
"em_relaxed_stderr": 0.015716934945725767
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:prehistory|0": {
|
|
"acc": 0.2852664576802508,
|
|
"acc_stderr": 0.025321176274679096,
|
|
"em_relaxed": 0.2852664576802508,
|
|
"em_relaxed_stderr": 0.025321176274679096
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:public_relations|0": {
|
|
"acc": 0.1651376146788991,
|
|
"acc_stderr": 0.0357288117600854,
|
|
"em_relaxed": 0.1651376146788991,
|
|
"em_relaxed_stderr": 0.0357288117600854
|
|
},
|
|
"e_tests.prompt3_mqa_multiple_correct_sk:obcianska-nauka|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:anatomy|0": {
|
|
"acc": 0.3656716417910448,
|
|
"acc_stderr": 0.041761576413228745,
|
|
"em_relaxed": 0.373134328358209,
|
|
"em_relaxed_stderr": 0.041936678130523646
|
|
},
|
|
"slovak_tests.prompt1_mqa_single_correct_sk:prijmacie_skusky|0": {
|
|
"acc": 0.21683673469387754,
|
|
"acc_stderr": 0.02084031144263498,
|
|
"em_relaxed": 0.2193877551020408,
|
|
"em_relaxed_stderr": 0.020928374259338522
|
|
},
|
|
"globalmmlu-lite.prompt1_mqa_single_correct_sk|0": {
|
|
"acc": 0.24,
|
|
"acc_stderr": 0.021380899352993928,
|
|
"em_relaxed": 0.2425,
|
|
"em_relaxed_stderr": 0.021456591955524264
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0": {
|
|
"acc": 0.30638297872340425,
|
|
"acc_stderr": 0.03013590647851756,
|
|
"em_relaxed": 0.3148936170212766,
|
|
"em_relaxed_stderr": 0.03036358219723817
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:philosophy|0": {
|
|
"acc": 0.22402597402597402,
|
|
"acc_stderr": 0.023795979800861743,
|
|
"em_relaxed": 0.22402597402597402,
|
|
"em_relaxed_stderr": 0.023795979800861743
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0": {
|
|
"acc": 0.2911392405063291,
|
|
"acc_stderr": 0.03625609398754825,
|
|
"em_relaxed": 0.2974683544303797,
|
|
"em_relaxed_stderr": 0.03648408889243667
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0": {
|
|
"acc": 0.19689119170984457,
|
|
"acc_stderr": 0.028697873971860688,
|
|
"em_relaxed": 0.19689119170984457,
|
|
"em_relaxed_stderr": 0.028697873971860688
|
|
},
|
|
"scio.prompt3_mqa_single_correct_sk:mat|0": {
|
|
"acc": 0.20765027322404372,
|
|
"acc_stderr": 0.021231382642567846,
|
|
"em_relaxed": 0.21584699453551912,
|
|
"em_relaxed_stderr": 0.02153411244027524
|
|
},
|
|
"e_tests.prompt2_qa_sk:geografia|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:public_relations|0": {
|
|
"acc": 0.21100917431192662,
|
|
"acc_stderr": 0.03926221238036852,
|
|
"em_relaxed": 0.21100917431192662,
|
|
"em_relaxed_stderr": 0.03926221238036852
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:marketing|0": {
|
|
"acc": 0.17094017094017094,
|
|
"acc_stderr": 0.02466249684520983,
|
|
"em_relaxed": 0.17094017094017094,
|
|
"em_relaxed_stderr": 0.02466249684520983
|
|
},
|
|
"scio.prompt1_mqa_single_correct_sk:bio|0": {
|
|
"acc": 0.2743362831858407,
|
|
"acc_stderr": 0.024268949584718702,
|
|
"em_relaxed": 0.28023598820059,
|
|
"em_relaxed_stderr": 0.024428605014452955
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0": {
|
|
"acc": 0.24352331606217617,
|
|
"acc_stderr": 0.03097543638684542,
|
|
"em_relaxed": 0.24870466321243523,
|
|
"em_relaxed_stderr": 0.0311958408777003
|
|
},
|
|
"sklep.prompt3_qa_sk:question-answering|0": {
|
|
"qem": 0.0002087029114056141,
|
|
"qem_stderr": 0.0001475675430580104,
|
|
"qf1": 0.006093522810123945,
|
|
"qf1_stderr": 0.00032767921496533014
|
|
},
|
|
"e_tests.prompt3_qa_sk:biologia|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0011204481792717084,
|
|
"rouge1_stderr": 0.0011204481792717086,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0011204481792717084,
|
|
"rougeL_stderr": 0.0011204481792717086
|
|
},
|
|
"slovak_tests.prompt3_mqa_single_correct_sk:vyssie_studium|0": {
|
|
"acc": 0.13333333333333333,
|
|
"acc_stderr": 0.03333333333333327,
|
|
"em_relaxed": 0.13333333333333333,
|
|
"em_relaxed_stderr": 0.03333333333333327
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:anatomy|0": {
|
|
"acc": 0.23134328358208955,
|
|
"acc_stderr": 0.03656527469759901,
|
|
"em_relaxed": 0.23134328358208955,
|
|
"em_relaxed_stderr": 0.03656527469759901
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:sociology|0": {
|
|
"acc": 0.2182741116751269,
|
|
"acc_stderr": 0.029505319811543015,
|
|
"em_relaxed": 0.2182741116751269,
|
|
"em_relaxed_stderr": 0.029505319811543015
|
|
},
|
|
"e_tests.prompt2_mqa_single_correct_sk:matematika|0": {
|
|
"acc": 0.10256410256410256,
|
|
"acc_stderr": 0.04921614717627658,
|
|
"em_relaxed": 0.3076923076923077,
|
|
"em_relaxed_stderr": 0.074871425137275
|
|
},
|
|
"scio.prompt1_mqa_single_correct_sk:aj|0": {
|
|
"acc": 0.6,
|
|
"acc_stderr": 0.16329931618554522,
|
|
"em_relaxed": 0.6,
|
|
"em_relaxed_stderr": 0.16329931618554522
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0": {
|
|
"acc": 0.22393822393822393,
|
|
"acc_stderr": 0.02595388693967799,
|
|
"em_relaxed": 0.23166023166023167,
|
|
"em_relaxed_stderr": 0.026265915716011466
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:global_facts|0": {
|
|
"acc": 0.2,
|
|
"acc_stderr": 0.04020151261036846,
|
|
"em_relaxed": 0.2,
|
|
"em_relaxed_stderr": 0.04020151261036846
|
|
},
|
|
"financial_exams.prompt1_completion_sk|0": {
|
|
"acc": 0.053973013493253376,
|
|
"acc_stderr": 0.006189064679835392,
|
|
"em_relaxed": 0.05697151424287856,
|
|
"em_relaxed_stderr": 0.006348574383669881
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0": {
|
|
"acc": 0.28110599078341014,
|
|
"acc_stderr": 0.03058726229145915,
|
|
"em_relaxed": 0.2903225806451613,
|
|
"em_relaxed_stderr": 0.030884745411494743
|
|
},
|
|
"slovak_tests.prompt3_qa_sk:prijmacie_skusky|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0012698412698412698,
|
|
"rouge1_stderr": 0.00126984126984127,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0012698412698412698,
|
|
"rougeL_stderr": 0.00126984126984127
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0": {
|
|
"acc": 0.22807017543859648,
|
|
"acc_stderr": 0.056069817847611766,
|
|
"em_relaxed": 0.22807017543859648,
|
|
"em_relaxed_stderr": 0.056069817847611766
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0": {
|
|
"acc": 0.2753623188405797,
|
|
"acc_stderr": 0.02693682252998795,
|
|
"em_relaxed": 0.27898550724637683,
|
|
"em_relaxed_stderr": 0.027045590734089176
|
|
},
|
|
"ifeval_sk.ifeval|0": {
|
|
"inst_level_loose_acc": 0.17312348668280872,
|
|
"inst_level_loose_acc_stderr": 0.0004210039776560394,
|
|
"inst_level_strict_acc": 0.17312348668280872,
|
|
"inst_level_strict_acc_stderr": 0.0004210039776560394,
|
|
"prompt_level_loose_acc": 0.0782122905027933,
|
|
"prompt_level_loose_acc_stderr": 0.011597665111537078,
|
|
"prompt_level_strict_acc": 0.0782122905027933,
|
|
"prompt_level_strict_acc_stderr": 0.011597665111537078
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0": {
|
|
"acc": 0.2826086956521739,
|
|
"acc_stderr": 0.027152165210427823,
|
|
"em_relaxed": 0.286231884057971,
|
|
"em_relaxed_stderr": 0.02725657169175206
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_law|0": {
|
|
"acc": 0.2314261315115286,
|
|
"acc_stderr": 0.012329801197790603,
|
|
"em_relaxed": 0.23825789923142612,
|
|
"em_relaxed_stderr": 0.012454741062931133
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:public_relations|0": {
|
|
"acc": 0.1834862385321101,
|
|
"acc_stderr": 0.03724531275523975,
|
|
"em_relaxed": 0.1834862385321101,
|
|
"em_relaxed_stderr": 0.03724531275523975
|
|
},
|
|
"movie_review.prompt1_sentiment_binary_sk|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slovak_tests.prompt1_qa_sk:monitor9|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0": {
|
|
"acc": 0.2416555407209613,
|
|
"acc_stderr": 0.01565239642514941,
|
|
"em_relaxed": 0.2416555407209613,
|
|
"em_relaxed_stderr": 0.01565239642514941
|
|
},
|
|
"slovak_tests.prompt1_mqa_single_correct_sk:maturita|0": {
|
|
"acc": 0.15718157181571815,
|
|
"acc_stderr": 0.018973345526861658,
|
|
"em_relaxed": 0.16260162601626016,
|
|
"em_relaxed_stderr": 0.019235549151947248
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0": {
|
|
"acc": 0.22897196261682243,
|
|
"acc_stderr": 0.028789653442089263,
|
|
"em_relaxed": 0.24766355140186916,
|
|
"em_relaxed_stderr": 0.02957653529316449
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:formal_logic|0": {
|
|
"acc": 0.17391304347826086,
|
|
"acc_stderr": 0.03549985134468377,
|
|
"em_relaxed": 0.20869565217391303,
|
|
"em_relaxed_stderr": 0.03806063686277675
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0": {
|
|
"acc": 0.21983914209115282,
|
|
"acc_stderr": 0.02147203067557281,
|
|
"em_relaxed": 0.21983914209115282,
|
|
"em_relaxed_stderr": 0.02147203067557281
|
|
},
|
|
"toxic_sk.prompt1_toxicity_sk|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"driving_test.prompt2_mqa_single_correct_sk|0": {
|
|
"acc": 0.09985734664764621,
|
|
"acc_stderr": 0.011331741573886145,
|
|
"em_relaxed": 0.10271041369472182,
|
|
"em_relaxed_stderr": 0.011474255943096628
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0": {
|
|
"acc": 0.32323232323232326,
|
|
"acc_stderr": 0.04724590344515123,
|
|
"em_relaxed": 0.3333333333333333,
|
|
"em_relaxed_stderr": 0.04761904761904759
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:astronomy|0": {
|
|
"acc": 0.19736842105263158,
|
|
"acc_stderr": 0.03238981601699397,
|
|
"em_relaxed": 0.21052631578947367,
|
|
"em_relaxed_stderr": 0.033176727875331574
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0": {
|
|
"acc": 0.2803030303030303,
|
|
"acc_stderr": 0.027695605168511307,
|
|
"em_relaxed": 0.2878787878787879,
|
|
"em_relaxed_stderr": 0.027919261062611087
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_law|0": {
|
|
"acc": 0.2322801024765158,
|
|
"acc_stderr": 0.012345664537818761,
|
|
"em_relaxed": 0.23654995730145176,
|
|
"em_relaxed_stderr": 0.012423924913578318
|
|
},
|
|
"sklep.prompt2_nli_3_classes_sk:nli|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0": {
|
|
"acc": 0.2119815668202765,
|
|
"acc_stderr": 0.02780931949344348,
|
|
"em_relaxed": 0.22580645161290322,
|
|
"em_relaxed_stderr": 0.028448938828651484
|
|
},
|
|
"sklep.prompt1_nli_binary_sk:rte|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"e_tests.prompt3_qa_sk:dejepis|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:computer_security|0": {
|
|
"acc": 0.22,
|
|
"acc_stderr": 0.0416333199893227,
|
|
"em_relaxed": 0.24,
|
|
"em_relaxed_stderr": 0.04292346959909283
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0": {
|
|
"acc": 0.2631578947368421,
|
|
"acc_stderr": 0.05884389414473131,
|
|
"em_relaxed": 0.2807017543859649,
|
|
"em_relaxed_stderr": 0.06004585739704729
|
|
},
|
|
"slovak_tests.prompt2_mqa_single_correct_sk:prijmacie_skusky|0": {
|
|
"acc": 0.13520408163265307,
|
|
"acc_stderr": 0.017292721140889967,
|
|
"em_relaxed": 0.1377551020408163,
|
|
"em_relaxed_stderr": 0.01742933367587574
|
|
},
|
|
"scio.prompt3_mqa_single_correct_sk:che|0": {
|
|
"acc": 0.22764227642276422,
|
|
"acc_stderr": 0.026788763286110986,
|
|
"em_relaxed": 0.23577235772357724,
|
|
"em_relaxed_stderr": 0.027119068752977968
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0": {
|
|
"acc": 0.23333333333333334,
|
|
"acc_stderr": 0.03464962582372412,
|
|
"em_relaxed": 0.23333333333333334,
|
|
"em_relaxed_stderr": 0.03464962582372412
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0": {
|
|
"acc": 0.1919191919191919,
|
|
"acc_stderr": 0.03978080447933685,
|
|
"em_relaxed": 0.1919191919191919,
|
|
"em_relaxed_stderr": 0.03978080447933685
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:econometrics|0": {
|
|
"acc": 0.16071428571428573,
|
|
"acc_stderr": 0.034859460964757394,
|
|
"em_relaxed": 0.16071428571428573,
|
|
"em_relaxed_stderr": 0.034859460964757394
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_biology|0": {
|
|
"acc": 0.2535211267605634,
|
|
"acc_stderr": 0.03663585085277138,
|
|
"em_relaxed": 0.2605633802816901,
|
|
"em_relaxed_stderr": 0.03696558732615873
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:nutrition|0": {
|
|
"acc": 0.22442244224422442,
|
|
"acc_stderr": 0.024007242951044432,
|
|
"em_relaxed": 0.25412541254125415,
|
|
"em_relaxed_stderr": 0.02505263910183634
|
|
},
|
|
"slovak_tests.prompt1_qa_sk:prijmacie_skusky|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.001330671989354624,
|
|
"rouge1_stderr": 0.0013306719893546237,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.001330671989354624,
|
|
"rougeL_stderr": 0.0013306719893546237
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0": {
|
|
"acc": 0.18711656441717792,
|
|
"acc_stderr": 0.021633588964875924,
|
|
"em_relaxed": 0.19325153374233128,
|
|
"em_relaxed_stderr": 0.02190225688747218
|
|
},
|
|
"e_tests.prompt2_mqa_single_correct_sk:slovensky-jazyk|0": {
|
|
"acc": 0.21615720524017468,
|
|
"acc_stderr": 0.0192548989397317,
|
|
"em_relaxed": 0.2183406113537118,
|
|
"em_relaxed_stderr": 0.01932493025894029
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:business_ethics|0": {
|
|
"acc": 0.14,
|
|
"acc_stderr": 0.0348735088019777,
|
|
"em_relaxed": 0.16,
|
|
"em_relaxed_stderr": 0.03684529491774708
|
|
},
|
|
"e_tests.prompt1_mqa_single_correct_sk:geografia|0": {
|
|
"acc": 0.20408163265306123,
|
|
"acc_stderr": 0.058172215566282555,
|
|
"em_relaxed": 0.20408163265306123,
|
|
"em_relaxed_stderr": 0.058172215566282555
|
|
},
|
|
"e_tests.prompt2_qa_sk:fyzika|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"multi_wiki_qa.prompt3_qa_sk|0": {
|
|
"qem": 0.0006003602161296778,
|
|
"qem_stderr": 0.00034654874631857635,
|
|
"qf1": 0.005765729879562343,
|
|
"qf1_stderr": 0.0005521722468476828
|
|
},
|
|
"e_tests.prompt1_qa_sk:fyzika|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0": {
|
|
"acc": 0.19172932330827067,
|
|
"acc_stderr": 0.024182419147367247,
|
|
"em_relaxed": 0.20676691729323307,
|
|
"em_relaxed_stderr": 0.024878145232561027
|
|
},
|
|
"e_tests.prompt2_mqa_single_correct_sk:obcianska-nauka|0": {
|
|
"acc": 0.15942028985507245,
|
|
"acc_stderr": 0.04439221265797129,
|
|
"em_relaxed": 0.15942028985507245,
|
|
"em_relaxed_stderr": 0.04439221265797129
|
|
},
|
|
"e_tests.prompt2_mqa_single_correct_sk:dejepis|0": {
|
|
"acc": 0.20833333333333334,
|
|
"acc_stderr": 0.059238169759886904,
|
|
"em_relaxed": 0.20833333333333334,
|
|
"em_relaxed_stderr": 0.059238169759886904
|
|
},
|
|
"scio.prompt1_mqa_single_correct_sk:vsp|0": {
|
|
"acc": 0.24150485436893204,
|
|
"acc_stderr": 0.01491898981873877,
|
|
"em_relaxed": 0.25849514563106796,
|
|
"em_relaxed_stderr": 0.015261010021381633
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:business_ethics|0": {
|
|
"acc": 0.16,
|
|
"acc_stderr": 0.03684529491774709,
|
|
"em_relaxed": 0.17,
|
|
"em_relaxed_stderr": 0.0377525168068637
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:international_law|0": {
|
|
"acc": 0.3050847457627119,
|
|
"acc_stderr": 0.04256799926288002,
|
|
"em_relaxed": 0.3050847457627119,
|
|
"em_relaxed_stderr": 0.04256799926288002
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:international_law|0": {
|
|
"acc": 0.288135593220339,
|
|
"acc_stderr": 0.04187011593049808,
|
|
"em_relaxed": 0.2966101694915254,
|
|
"em_relaxed_stderr": 0.04222776832233627
|
|
},
|
|
"slovakcopa.prompt2_copa_sk|0": {
|
|
"acc": 0.006,
|
|
"acc_stderr": 0.003457152557758384,
|
|
"em_relaxed": 0.006,
|
|
"em_relaxed_stderr": 0.003457152557758384
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0": {
|
|
"acc": 0.2116564417177914,
|
|
"acc_stderr": 0.022658534978587233,
|
|
"em_relaxed": 0.22085889570552147,
|
|
"em_relaxed_stderr": 0.023010381863796215
|
|
},
|
|
"e_tests.prompt2_mqa_multiple_correct_sk:fyzika|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:global_facts|0": {
|
|
"acc": 0.24,
|
|
"acc_stderr": 0.042923469599092816,
|
|
"em_relaxed": 0.25,
|
|
"em_relaxed_stderr": 0.04351941398892446
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:world_religions|0": {
|
|
"acc": 0.2807017543859649,
|
|
"acc_stderr": 0.034462962170884265,
|
|
"em_relaxed": 0.2807017543859649,
|
|
"em_relaxed_stderr": 0.034462962170884265
|
|
},
|
|
"e_tests.prompt1_qa_sk:biologia|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0": {
|
|
"acc": 0.15,
|
|
"acc_stderr": 0.03588702812826371,
|
|
"em_relaxed": 0.15,
|
|
"em_relaxed_stderr": 0.03588702812826371
|
|
},
|
|
"e_tests.prompt3_mqa_single_correct_sk:chemia|0": {
|
|
"acc": 0.07142857142857142,
|
|
"acc_stderr": 0.03472660248602842,
|
|
"em_relaxed": 0.08928571428571429,
|
|
"em_relaxed_stderr": 0.038450387280282494
|
|
},
|
|
"e_tests.prompt1_mqa_single_correct_sk:matematika|0": {
|
|
"acc": 0.15384615384615385,
|
|
"acc_stderr": 0.05852968540405249,
|
|
"em_relaxed": 0.358974358974359,
|
|
"em_relaxed_stderr": 0.07781756136754926
|
|
},
|
|
"e_tests.prompt3_qa_sk:geografia|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"e_tests.prompt2_mqa_multiple_correct_sk:chemia|0": {
|
|
"acc": 0.045454545454545456,
|
|
"acc_stderr": 0.031765283723714
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0": {
|
|
"acc": 0.2383177570093458,
|
|
"acc_stderr": 0.029192770642101563,
|
|
"em_relaxed": 0.2570093457943925,
|
|
"em_relaxed_stderr": 0.029941691533244646
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0": {
|
|
"acc": 0.24932975871313673,
|
|
"acc_stderr": 0.022430563714050267,
|
|
"em_relaxed": 0.24932975871313673,
|
|
"em_relaxed_stderr": 0.022430563714050267
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0": {
|
|
"acc": 0.24468085106382978,
|
|
"acc_stderr": 0.025645553622266722,
|
|
"em_relaxed": 0.25177304964539005,
|
|
"em_relaxed_stderr": 0.025892151156709405
|
|
},
|
|
"e_tests.prompt1_mqa_multiple_correct_sk:informatika|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_biology|0": {
|
|
"acc": 0.22535211267605634,
|
|
"acc_stderr": 0.035186286213623755,
|
|
"em_relaxed": 0.22535211267605634,
|
|
"em_relaxed_stderr": 0.035186286213623755
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0": {
|
|
"acc": 0.296551724137931,
|
|
"acc_stderr": 0.03806142687309993,
|
|
"em_relaxed": 0.3103448275862069,
|
|
"em_relaxed_stderr": 0.03855289616378948
|
|
},
|
|
"e_tests.prompt1_mqa_single_correct_sk:chemia|0": {
|
|
"acc": 0.10714285714285714,
|
|
"acc_stderr": 0.0417053005800816,
|
|
"em_relaxed": 0.10714285714285714,
|
|
"em_relaxed_stderr": 0.0417053005800816
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0": {
|
|
"acc": 0.25671140939597314,
|
|
"acc_stderr": 0.01790782476183757,
|
|
"em_relaxed": 0.25838926174496646,
|
|
"em_relaxed_stderr": 0.017945962358551722
|
|
},
|
|
"e_tests.prompt3_mqa_multiple_correct_sk:chemia|0": {
|
|
"acc": 0.045454545454545456,
|
|
"acc_stderr": 0.031765283723714
|
|
},
|
|
"e_tests.prompt3_qa_sk:obcianska-nauka|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:computer_security|0": {
|
|
"acc": 0.2,
|
|
"acc_stderr": 0.04020151261036843,
|
|
"em_relaxed": 0.22,
|
|
"em_relaxed_stderr": 0.041633319989322716
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0": {
|
|
"acc": 0.2018348623853211,
|
|
"acc_stderr": 0.03862180715692686,
|
|
"em_relaxed": 0.2018348623853211,
|
|
"em_relaxed_stderr": 0.03862180715692686
|
|
},
|
|
"e_tests.prompt3_mqa_single_correct_sk:obcianska-nauka|0": {
|
|
"acc": 0.14492753623188406,
|
|
"acc_stderr": 0.0426896366873121,
|
|
"em_relaxed": 0.14492753623188406,
|
|
"em_relaxed_stderr": 0.0426896366873121
|
|
},
|
|
"e_tests.prompt3_mqa_single_correct_sk:fyzika|0": {
|
|
"acc": 0.18181818181818182,
|
|
"acc_stderr": 0.08416546361568647,
|
|
"em_relaxed": 0.2727272727272727,
|
|
"em_relaxed_stderr": 0.0971859061499725
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0": {
|
|
"acc": 0.16917293233082706,
|
|
"acc_stderr": 0.023030205866522328,
|
|
"em_relaxed": 0.17669172932330826,
|
|
"em_relaxed_stderr": 0.023429683004809066
|
|
},
|
|
"e_tests.prompt2_mqa_multiple_correct_sk:dejepis|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"e_tests.prompt2_mqa_multiple_correct_sk:geografia|0": {
|
|
"acc": 0.034482758620689655,
|
|
"acc_stderr": 0.034482758620689676
|
|
},
|
|
"e_tests.prompt3_qa_sk:chemia|0": {
|
|
"qem": 0.018518518518518517,
|
|
"qem_stderr": 0.018518518518518517,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0006660882976672449,
|
|
"rouge1_stderr": 0.0004499203448417917,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0006660882976672449,
|
|
"rougeL_stderr": 0.0004499203448417917
|
|
},
|
|
"e_tests.prompt3_mqa_multiple_correct_sk:biologia|0": {
|
|
"acc": 0.03571428571428571,
|
|
"acc_stderr": 0.02502318034804895
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:formal_logic|0": {
|
|
"acc": 0.1826086956521739,
|
|
"acc_stderr": 0.03618456100987082,
|
|
"em_relaxed": 0.21739130434782608,
|
|
"em_relaxed_stderr": 0.038631448549506
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0": {
|
|
"acc": 0.21379310344827587,
|
|
"acc_stderr": 0.034165204477475494,
|
|
"em_relaxed": 0.2206896551724138,
|
|
"em_relaxed_stderr": 0.03455930201924811
|
|
},
|
|
"e_tests.prompt2_mqa_multiple_correct_sk:slovensky-jazyk|0": {
|
|
"acc": 0.014285714285714285,
|
|
"acc_stderr": 0.010065123433632259
|
|
},
|
|
"e_tests.prompt1_mqa_multiple_correct_sk:matematika|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0": {
|
|
"acc": 0.22,
|
|
"acc_stderr": 0.041633319989322695,
|
|
"em_relaxed": 0.24,
|
|
"em_relaxed_stderr": 0.042923469599092816
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:marketing|0": {
|
|
"acc": 0.24786324786324787,
|
|
"acc_stderr": 0.028286324075564397,
|
|
"em_relaxed": 0.25213675213675213,
|
|
"em_relaxed_stderr": 0.02844796547623102
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:world_religions|0": {
|
|
"acc": 0.26900584795321636,
|
|
"acc_stderr": 0.0340105262010409,
|
|
"em_relaxed": 0.27485380116959063,
|
|
"em_relaxed_stderr": 0.034240429246915824
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0": {
|
|
"acc": 0.21717171717171718,
|
|
"acc_stderr": 0.02937661648494564,
|
|
"em_relaxed": 0.21717171717171718,
|
|
"em_relaxed_stderr": 0.02937661648494564
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0": {
|
|
"acc": 0.26702269692923897,
|
|
"acc_stderr": 0.01617590372442817,
|
|
"em_relaxed": 0.26702269692923897,
|
|
"em_relaxed_stderr": 0.01617590372442817
|
|
},
|
|
"e_tests.prompt2_qa_sk:informatika|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.010073260073260074,
|
|
"qf1_stderr": 0.007341600122632042,
|
|
"rouge1": 0.0012475950143262986,
|
|
"rouge1_stderr": 0.000686304468853021,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0012475950143262986,
|
|
"rougeL_stderr": 0.000686304468853021
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0": {
|
|
"acc": 0.2474747474747475,
|
|
"acc_stderr": 0.030746300742124498,
|
|
"em_relaxed": 0.25252525252525254,
|
|
"em_relaxed_stderr": 0.030954055470365897
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0": {
|
|
"acc": 0.24242424242424243,
|
|
"acc_stderr": 0.04329004329004326,
|
|
"em_relaxed": 0.26262626262626265,
|
|
"em_relaxed_stderr": 0.04445287676983944
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:machine_learning|0": {
|
|
"acc": 0.16964285714285715,
|
|
"acc_stderr": 0.0356236785009539,
|
|
"em_relaxed": 0.19642857142857142,
|
|
"em_relaxed_stderr": 0.03770970049347018
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:virology|0": {
|
|
"acc": 0.22560975609756098,
|
|
"acc_stderr": 0.03273897454566342,
|
|
"em_relaxed": 0.22560975609756098,
|
|
"em_relaxed_stderr": 0.03273897454566342
|
|
},
|
|
"slovakcopa.prompt1_copa_sk|0": {
|
|
"acc": 0.01,
|
|
"acc_stderr": 0.0044541754970721,
|
|
"em_relaxed": 0.01,
|
|
"em_relaxed_stderr": 0.0044541754970721
|
|
},
|
|
"scio.prompt3_mqa_single_correct_sk:bio|0": {
|
|
"acc": 0.3008849557522124,
|
|
"acc_stderr": 0.024946878423156852,
|
|
"em_relaxed": 0.30973451327433627,
|
|
"em_relaxed_stderr": 0.025150378763664
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0": {
|
|
"acc": 0.205607476635514,
|
|
"acc_stderr": 0.03925401580070484,
|
|
"em_relaxed": 0.21495327102803738,
|
|
"em_relaxed_stderr": 0.03989944463395407
|
|
},
|
|
"multi_wiki_qa.prompt2_qa_sk|0": {
|
|
"qem": 0.0006003602161296778,
|
|
"qem_stderr": 0.0003465487463185779,
|
|
"qf1": 0.005417769336679904,
|
|
"qf1_stderr": 0.0005509525930025259
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_physics|0": {
|
|
"acc": 0.21568627450980393,
|
|
"acc_stderr": 0.04092563958237654,
|
|
"em_relaxed": 0.22549019607843138,
|
|
"em_relaxed_stderr": 0.04158307533083286
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0": {
|
|
"acc": 0.20425531914893616,
|
|
"acc_stderr": 0.026355158413349424,
|
|
"em_relaxed": 0.20425531914893616,
|
|
"em_relaxed_stderr": 0.026355158413349424
|
|
},
|
|
"e_tests.prompt1_mqa_single_correct_sk:obcianska-nauka|0": {
|
|
"acc": 0.18840579710144928,
|
|
"acc_stderr": 0.047420064740574204,
|
|
"em_relaxed": 0.18840579710144928,
|
|
"em_relaxed_stderr": 0.047420064740574204
|
|
},
|
|
"e_tests.prompt3_mqa_single_correct_sk:dejepis|0": {
|
|
"acc": 0.20833333333333334,
|
|
"acc_stderr": 0.059238169759886904,
|
|
"em_relaxed": 0.20833333333333334,
|
|
"em_relaxed_stderr": 0.059238169759886904
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:prehistory|0": {
|
|
"acc": 0.19122257053291536,
|
|
"acc_stderr": 0.022053144718757862,
|
|
"em_relaxed": 0.19122257053291536,
|
|
"em_relaxed_stderr": 0.022053144718757862
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0": {
|
|
"acc": 0.23333333333333334,
|
|
"acc_stderr": 0.03464962582372412,
|
|
"em_relaxed": 0.23333333333333334,
|
|
"em_relaxed_stderr": 0.03464962582372412
|
|
},
|
|
"sklep.prompt1_hate_speech_sk:hate-speech|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"driving_test.prompt1_mqa_single_correct_sk|0": {
|
|
"acc": 0.16405135520684735,
|
|
"acc_stderr": 0.01399686230459443,
|
|
"em_relaxed": 0.16833095577746077,
|
|
"em_relaxed_stderr": 0.014141915666582279
|
|
},
|
|
"e_tests.prompt3_mqa_single_correct_sk:informatika|0": {
|
|
"acc": 0.256,
|
|
"acc_stderr": 0.03919183588453089,
|
|
"em_relaxed": 0.264,
|
|
"em_relaxed_stderr": 0.03958494337416042
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0": {
|
|
"acc": 0.18959731543624161,
|
|
"acc_stderr": 0.01606971018943585,
|
|
"em_relaxed": 0.19463087248322147,
|
|
"em_relaxed_stderr": 0.0162309847870997
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0": {
|
|
"acc": 0.22429906542056074,
|
|
"acc_stderr": 0.028580583273338626,
|
|
"em_relaxed": 0.2383177570093458,
|
|
"em_relaxed_stderr": 0.029192770642101563
|
|
},
|
|
"e_tests.prompt1_mqa_single_correct_sk:fyzika|0": {
|
|
"acc": 0.13636363636363635,
|
|
"acc_stderr": 0.07488677009526491,
|
|
"em_relaxed": 0.22727272727272727,
|
|
"em_relaxed_stderr": 0.09144861547306321
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_medicine|0": {
|
|
"acc": 0.26744186046511625,
|
|
"acc_stderr": 0.03384836428157859,
|
|
"em_relaxed": 0.27906976744186046,
|
|
"em_relaxed_stderr": 0.03430085607014882
|
|
},
|
|
"slovak_tests.prompt3_qa_sk:monitor9|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:human_aging|0": {
|
|
"acc": 0.136986301369863,
|
|
"acc_stderr": 0.023287301596102452,
|
|
"em_relaxed": 0.136986301369863,
|
|
"em_relaxed_stderr": 0.023287301596102452
|
|
},
|
|
"globalmmlu-lite.prompt3_mqa_single_correct_sk|0": {
|
|
"acc": 0.2025,
|
|
"acc_stderr": 0.0201183061066658,
|
|
"em_relaxed": 0.205,
|
|
"em_relaxed_stderr": 0.02021035988339997
|
|
},
|
|
"sklep.prompt2_qa_sk:question-answering|0": {
|
|
"qem": 0.0002087029114056141,
|
|
"qem_stderr": 0.0001475675430580104,
|
|
"qf1": 0.006743145308527306,
|
|
"qf1_stderr": 0.00039734127192685853
|
|
},
|
|
"truthfulqa.prompt2_mqa_single_correct_sk:mc1|0": {
|
|
"acc": 0.04884318766066838,
|
|
"acc_stderr": 0.00773245768018304,
|
|
"em_relaxed": 0.08354755784061697,
|
|
"em_relaxed_stderr": 0.009926847196525902
|
|
},
|
|
"senti_sk.prompt3_sentiment_binary_sk|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"scio.prompt2_mqa_single_correct_sk:vsp|0": {
|
|
"acc": 0.23786407766990292,
|
|
"acc_stderr": 0.014841600192116697,
|
|
"em_relaxed": 0.2512135922330097,
|
|
"em_relaxed_stderr": 0.015118218765838777
|
|
},
|
|
"e_tests.prompt3_mqa_multiple_correct_sk:matematika|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"e_tests.prompt3_qa_sk:slovensky-jazyk|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0025906735751295338,
|
|
"qf1_stderr": 0.0025906735751295403,
|
|
"rouge1": 0.00035708568846413394,
|
|
"rouge1_stderr": 0.00020841637674444064,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.00035708568846413394,
|
|
"rougeL_stderr": 0.00020841637674444064
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0": {
|
|
"acc": 0.24561403508771928,
|
|
"acc_stderr": 0.05752139056405265,
|
|
"em_relaxed": 0.2631578947368421,
|
|
"em_relaxed_stderr": 0.05884389414473131
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0": {
|
|
"acc": 0.16595744680851063,
|
|
"acc_stderr": 0.02432117475103868,
|
|
"em_relaxed": 0.16595744680851063,
|
|
"em_relaxed_stderr": 0.02432117475103868
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0": {
|
|
"acc": 0.2126865671641791,
|
|
"acc_stderr": 0.01769159425053082,
|
|
"em_relaxed": 0.21641791044776118,
|
|
"em_relaxed_stderr": 0.01780376934180082
|
|
},
|
|
"senti_sk.prompt2_sentiment_binary_sk|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"scio.prompt1_mqa_single_correct_sk:che|0": {
|
|
"acc": 0.25609756097560976,
|
|
"acc_stderr": 0.027885450835842325,
|
|
"em_relaxed": 0.25609756097560976,
|
|
"em_relaxed_stderr": 0.027885450835842325
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0": {
|
|
"acc": 0.24444444444444444,
|
|
"acc_stderr": 0.03212157057535212,
|
|
"em_relaxed": 0.25,
|
|
"em_relaxed_stderr": 0.032364888900157734
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:machine_learning|0": {
|
|
"acc": 0.14285714285714285,
|
|
"acc_stderr": 0.03321361106966269,
|
|
"em_relaxed": 0.16071428571428573,
|
|
"em_relaxed_stderr": 0.0348594609647574
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0": {
|
|
"acc": 0.21212121212121213,
|
|
"acc_stderr": 0.04129606932540889,
|
|
"em_relaxed": 0.24242424242424243,
|
|
"em_relaxed_stderr": 0.04329004329004327
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:prehistory|0": {
|
|
"acc": 0.26959247648902823,
|
|
"acc_stderr": 0.024884157592464488,
|
|
"em_relaxed": 0.26959247648902823,
|
|
"em_relaxed_stderr": 0.024884157592464488
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0": {
|
|
"acc": 0.2538860103626943,
|
|
"acc_stderr": 0.03141024780565318,
|
|
"em_relaxed": 0.2538860103626943,
|
|
"em_relaxed_stderr": 0.03141024780565318
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0": {
|
|
"acc": 0.21212121212121213,
|
|
"acc_stderr": 0.04129606932540889,
|
|
"em_relaxed": 0.23232323232323232,
|
|
"em_relaxed_stderr": 0.042660160170546885
|
|
},
|
|
"truthfulqa.prompt1_mqa_single_correct_sk:mc1|0": {
|
|
"acc": 0.042416452442159386,
|
|
"acc_stderr": 0.0072301115841424965,
|
|
"em_relaxed": 0.07840616966580977,
|
|
"em_relaxed_stderr": 0.009643493430258573
|
|
},
|
|
"belebele.prompt3_mqa_single_correct_sk|0": {
|
|
"acc": 0.24666666666666667,
|
|
"acc_stderr": 0.014377023375409347,
|
|
"em_relaxed": 0.2544444444444444,
|
|
"em_relaxed_stderr": 0.014526354751055178
|
|
},
|
|
"arc.prompt3_mqa_single_correct_sk|0": {
|
|
"acc": 0.22840034217279725,
|
|
"acc_stderr": 0.012283523666042804,
|
|
"em_relaxed": 0.22840034217279725,
|
|
"em_relaxed_stderr": 0.012283523666042804
|
|
},
|
|
"financial_exams.prompt2_completion_sk|0": {
|
|
"acc": 0.07871064467766117,
|
|
"acc_stderr": 0.00737563958812383,
|
|
"em_relaxed": 0.07946026986506746,
|
|
"em_relaxed_stderr": 0.007407662917614668
|
|
},
|
|
"e_tests.prompt2_mqa_multiple_correct_sk:matematika|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0": {
|
|
"acc": 0.23049645390070922,
|
|
"acc_stderr": 0.025123739226872395,
|
|
"em_relaxed": 0.24113475177304963,
|
|
"em_relaxed_stderr": 0.02551873104953778
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0": {
|
|
"acc": 0.24,
|
|
"acc_stderr": 0.042923469599092816,
|
|
"em_relaxed": 0.24,
|
|
"em_relaxed_stderr": 0.042923469599092816
|
|
},
|
|
"e_tests.prompt3_mqa_single_correct_sk:biologia|0": {
|
|
"acc": 0.10344827586206896,
|
|
"acc_stderr": 0.03283976780985287,
|
|
"em_relaxed": 0.11494252873563218,
|
|
"em_relaxed_stderr": 0.03439354046699091
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0": {
|
|
"acc": 0.24299065420560748,
|
|
"acc_stderr": 0.041657429989652724,
|
|
"em_relaxed": 0.24299065420560748,
|
|
"em_relaxed_stderr": 0.041657429989652724
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:human_aging|0": {
|
|
"acc": 0.2191780821917808,
|
|
"acc_stderr": 0.028018592326462144,
|
|
"em_relaxed": 0.2191780821917808,
|
|
"em_relaxed_stderr": 0.028018592326462144
|
|
},
|
|
"slovak_tests.prompt3_mqa_single_correct_sk:maturita|0": {
|
|
"acc": 0.18157181571815717,
|
|
"acc_stderr": 0.020095119692986566,
|
|
"em_relaxed": 0.18157181571815717,
|
|
"em_relaxed_stderr": 0.020095119692986566
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_physics|0": {
|
|
"acc": 0.22549019607843138,
|
|
"acc_stderr": 0.04158307533083286,
|
|
"em_relaxed": 0.23529411764705882,
|
|
"em_relaxed_stderr": 0.04220773659171455
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:econometrics|0": {
|
|
"acc": 0.20535714285714285,
|
|
"acc_stderr": 0.03834241021419071,
|
|
"em_relaxed": 0.22321428571428573,
|
|
"em_relaxed_stderr": 0.039523019677025116
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0": {
|
|
"acc": 0.1921641791044776,
|
|
"acc_stderr": 0.01703416618213854,
|
|
"em_relaxed": 0.1958955223880597,
|
|
"em_relaxed_stderr": 0.01715898536836359
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_physics|0": {
|
|
"acc": 0.23529411764705882,
|
|
"acc_stderr": 0.04220773659171455,
|
|
"em_relaxed": 0.24509803921568626,
|
|
"em_relaxed_stderr": 0.042801058373643945
|
|
},
|
|
"e_tests.prompt1_mqa_single_correct_sk:dejepis|0": {
|
|
"acc": 0.22916666666666666,
|
|
"acc_stderr": 0.06130657275434794,
|
|
"em_relaxed": 0.22916666666666666,
|
|
"em_relaxed_stderr": 0.06130657275434794
|
|
},
|
|
"scio.prompt2_mqa_single_correct_sk:bio|0": {
|
|
"acc": 0.2861356932153392,
|
|
"acc_stderr": 0.02458303513041921,
|
|
"em_relaxed": 0.2949852507374631,
|
|
"em_relaxed_stderr": 0.024805095426679618
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0": {
|
|
"acc": 0.23255813953488372,
|
|
"acc_stderr": 0.021502792400356754,
|
|
"em_relaxed": 0.24031007751937986,
|
|
"em_relaxed_stderr": 0.02174755921996615
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0": {
|
|
"acc": 0.2537878787878788,
|
|
"acc_stderr": 0.02683420376859365,
|
|
"em_relaxed": 0.25757575757575757,
|
|
"em_relaxed_stderr": 0.026965016533893105
|
|
},
|
|
"toxic_sk.prompt3_toxicity_sk|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0": {
|
|
"acc": 0.17777777777777778,
|
|
"acc_stderr": 0.033027898599017176,
|
|
"em_relaxed": 0.17777777777777778,
|
|
"em_relaxed_stderr": 0.033027898599017176
|
|
},
|
|
"driving_test.prompt3_mqa_single_correct_sk|0": {
|
|
"acc": 0.14835948644793154,
|
|
"acc_stderr": 0.013434972977209091,
|
|
"em_relaxed": 0.15121255349500715,
|
|
"em_relaxed_stderr": 0.01354080168340582
|
|
},
|
|
"slovak_tests.prompt3_mqa_single_correct_sk:prijmacie_skusky|0": {
|
|
"acc": 0.1760204081632653,
|
|
"acc_stderr": 0.019259781722629446,
|
|
"em_relaxed": 0.1760204081632653,
|
|
"em_relaxed_stderr": 0.019259781722629446
|
|
},
|
|
"globalmmlu-lite.prompt2_mqa_single_correct_sk|0": {
|
|
"acc": 0.19,
|
|
"acc_stderr": 0.019639610121239284,
|
|
"em_relaxed": 0.1925,
|
|
"em_relaxed_stderr": 0.01973786547054499
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:philosophy|0": {
|
|
"acc": 0.12337662337662338,
|
|
"acc_stderr": 0.018769546352956558,
|
|
"em_relaxed": 0.12337662337662338,
|
|
"em_relaxed_stderr": 0.018769546352956558
|
|
},
|
|
"sklep.prompt1_qa_sk:question-answering|0": {
|
|
"qem": 0.0004174058228112282,
|
|
"qem_stderr": 0.00020867023775979663,
|
|
"qf1": 0.008276210006991061,
|
|
"qf1_stderr": 0.00042241089189288785
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:business_ethics|0": {
|
|
"acc": 0.2,
|
|
"acc_stderr": 0.04020151261036846,
|
|
"em_relaxed": 0.22,
|
|
"em_relaxed_stderr": 0.041633319989322674
|
|
},
|
|
"skj1.prompt1_skj1_sk|0": {
|
|
"acc": 0.41,
|
|
"acc_stderr": 0.04943110704237102,
|
|
"em_relaxed": 0.44,
|
|
"em_relaxed_stderr": 0.049888765156985884
|
|
},
|
|
"e_tests.prompt2_qa_sk:slovensky-jazyk|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0025906735751295338,
|
|
"qf1_stderr": 0.0025906735751295403,
|
|
"rouge1": 0.00011405864075384238,
|
|
"rouge1_stderr": 8.908193183205394e-05,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.00011405864075384238,
|
|
"rougeL_stderr": 8.908193183205394e-05
|
|
},
|
|
"belebele.prompt1_mqa_single_correct_sk|0": {
|
|
"acc": 0.2633333333333333,
|
|
"acc_stderr": 0.014689553047342515,
|
|
"em_relaxed": 0.2722222222222222,
|
|
"em_relaxed_stderr": 0.014845038794433947
|
|
},
|
|
"e_tests.prompt2_qa_sk:chemia|0": {
|
|
"qem": 0.018518518518518517,
|
|
"qem_stderr": 0.018518518518518517,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 3.292181069958847e-05,
|
|
"rouge1_stderr": 3.292181069958849e-05,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 3.292181069958847e-05,
|
|
"rougeL_stderr": 3.292181069958849e-05
|
|
},
|
|
"sklep.prompt2_nli_binary_sk:rte|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"multi_wiki_qa.prompt1_qa_sk|0": {
|
|
"qem": 0.00040024014408645187,
|
|
"qem_stderr": 0.00028298419465801113,
|
|
"qf1": 0.004608242571473309,
|
|
"qf1_stderr": 0.00046536745363106924
|
|
},
|
|
"slovak_tests.prompt1_mqa_single_correct_sk:vyssie_studium|0": {
|
|
"acc": 0.21904761904761905,
|
|
"acc_stderr": 0.040556911537178254,
|
|
"em_relaxed": 0.21904761904761905,
|
|
"em_relaxed_stderr": 0.040556911537178254
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0": {
|
|
"acc": 0.17,
|
|
"acc_stderr": 0.03775251680686371,
|
|
"em_relaxed": 0.17,
|
|
"em_relaxed_stderr": 0.03775251680686371
|
|
},
|
|
"skj1.prompt3_skj1_sk|0": {
|
|
"acc": 0.75,
|
|
"acc_stderr": 0.04351941398892446,
|
|
"em_relaxed": 0.75,
|
|
"em_relaxed_stderr": 0.04351941398892446
|
|
},
|
|
"slovak_tests.prompt2_qa_sk:prijmacie_skusky|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0021574973031283713,
|
|
"rouge1_stderr": 0.0021574973031283713,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0010787486515641857,
|
|
"rougeL_stderr": 0.0010787486515641857
|
|
},
|
|
"e_tests.prompt1_mqa_multiple_correct_sk:geografia|0": {
|
|
"acc": 0.034482758620689655,
|
|
"acc_stderr": 0.034482758620689676
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0": {
|
|
"acc": 0.22777777777777777,
|
|
"acc_stderr": 0.03134731210385172,
|
|
"em_relaxed": 0.23333333333333334,
|
|
"em_relaxed_stderr": 0.031612960436120094
|
|
},
|
|
"scio.prompt2_mqa_single_correct_sk:mat|0": {
|
|
"acc": 0.21311475409836064,
|
|
"acc_stderr": 0.021434631795998253,
|
|
"em_relaxed": 0.22131147540983606,
|
|
"em_relaxed_stderr": 0.021728884019137824
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_medicine|0": {
|
|
"acc": 0.20930232558139536,
|
|
"acc_stderr": 0.03110958390976464,
|
|
"em_relaxed": 0.21511627906976744,
|
|
"em_relaxed_stderr": 0.03142253684735939
|
|
},
|
|
"sklep.prompt3_hate_speech_sk:hate-speech|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0": {
|
|
"acc": 0.18666666666666668,
|
|
"acc_stderr": 0.03192085664169663,
|
|
"em_relaxed": 0.19333333333333333,
|
|
"em_relaxed_stderr": 0.03235245865324252
|
|
},
|
|
"scio.prompt2_mqa_single_correct_sk:che|0": {
|
|
"acc": 0.18699186991869918,
|
|
"acc_stderr": 0.024910116828719235,
|
|
"em_relaxed": 0.1910569105691057,
|
|
"em_relaxed_stderr": 0.025116395774587044
|
|
},
|
|
"skj1.prompt2_skj1_sk|0": {
|
|
"acc": 0.63,
|
|
"acc_stderr": 0.04852365870939099,
|
|
"em_relaxed": 0.64,
|
|
"em_relaxed_stderr": 0.048241815132442176
|
|
},
|
|
"truthfulqa.prompt3_mqa_single_correct_sk:mc1|0": {
|
|
"acc": 0.10282776349614396,
|
|
"acc_stderr": 0.010896388982494143,
|
|
"em_relaxed": 0.12853470437017994,
|
|
"em_relaxed_stderr": 0.012006729887990094
|
|
},
|
|
"e_tests.prompt3_qa_sk:informatika|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.006923076923076923,
|
|
"qf1_stderr": 0.006424094288133824,
|
|
"rouge1": 0.00046479944765467505,
|
|
"rouge1_stderr": 0.0001876188412456943,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.00046479944765467505,
|
|
"rougeL_stderr": 0.0001876188412456943
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:astronomy|0": {
|
|
"acc": 0.23026315789473684,
|
|
"acc_stderr": 0.03426059424403165,
|
|
"em_relaxed": 0.23026315789473684,
|
|
"em_relaxed_stderr": 0.03426059424403165
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:computer_security|0": {
|
|
"acc": 0.27,
|
|
"acc_stderr": 0.04461960433384741,
|
|
"em_relaxed": 0.28,
|
|
"em_relaxed_stderr": 0.04512608598542127
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:nutrition|0": {
|
|
"acc": 0.2145214521452145,
|
|
"acc_stderr": 0.023621042274383643,
|
|
"em_relaxed": 0.24092409240924093,
|
|
"em_relaxed_stderr": 0.0246081660183802
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0": {
|
|
"acc": 0.2676767676767677,
|
|
"acc_stderr": 0.03154449888270286,
|
|
"em_relaxed": 0.2676767676767677,
|
|
"em_relaxed_stderr": 0.03154449888270286
|
|
},
|
|
"financial_exams.prompt3_completion_sk|0": {
|
|
"acc": 0.13343328335832083,
|
|
"acc_stderr": 0.009313609263721821,
|
|
"em_relaxed": 0.13343328335832083,
|
|
"em_relaxed_stderr": 0.009313609263721821
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:sociology|0": {
|
|
"acc": 0.19289340101522842,
|
|
"acc_stderr": 0.028183576131702342,
|
|
"em_relaxed": 0.19289340101522842,
|
|
"em_relaxed_stderr": 0.028183576131702342
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:world_religions|0": {
|
|
"acc": 0.26900584795321636,
|
|
"acc_stderr": 0.0340105262010409,
|
|
"em_relaxed": 0.2807017543859649,
|
|
"em_relaxed_stderr": 0.034462962170884265
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0": {
|
|
"acc": 0.15555555555555556,
|
|
"acc_stderr": 0.03130948364878314,
|
|
"em_relaxed": 0.15555555555555556,
|
|
"em_relaxed_stderr": 0.03130948364878314
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0": {
|
|
"acc": 0.18,
|
|
"acc_stderr": 0.03861229196653695,
|
|
"em_relaxed": 0.18,
|
|
"em_relaxed_stderr": 0.03861229196653695
|
|
},
|
|
"scio.prompt3_mqa_single_correct_sk:aj|0": {
|
|
"acc": 0.5,
|
|
"acc_stderr": 0.16666666666666666,
|
|
"em_relaxed": 0.5,
|
|
"em_relaxed_stderr": 0.16666666666666666
|
|
},
|
|
"e_tests.prompt3_mqa_single_correct_sk:geografia|0": {
|
|
"acc": 0.22448979591836735,
|
|
"acc_stderr": 0.06022425581505364,
|
|
"em_relaxed": 0.22448979591836735,
|
|
"em_relaxed_stderr": 0.06022425581505364
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0": {
|
|
"acc": 0.18181818181818182,
|
|
"acc_stderr": 0.03896103896103899,
|
|
"em_relaxed": 0.18181818181818182,
|
|
"em_relaxed_stderr": 0.03896103896103899
|
|
},
|
|
"e_tests.prompt1_mqa_multiple_correct_sk:fyzika|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:sociology|0": {
|
|
"acc": 0.15228426395939088,
|
|
"acc_stderr": 0.02566401503870399,
|
|
"em_relaxed": 0.15228426395939088,
|
|
"em_relaxed_stderr": 0.02566401503870399
|
|
},
|
|
"e_tests.prompt1_qa_sk:obcianska-nauka|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"e_tests.prompt3_mqa_single_correct_sk:matematika|0": {
|
|
"acc": 0.05128205128205128,
|
|
"acc_stderr": 0.035781575124326184,
|
|
"em_relaxed": 0.23076923076923078,
|
|
"em_relaxed_stderr": 0.0683479474337414
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0": {
|
|
"acc": 0.26515151515151514,
|
|
"acc_stderr": 0.0272187441820065,
|
|
"em_relaxed": 0.2765151515151515,
|
|
"em_relaxed_stderr": 0.027580130232496353
|
|
},
|
|
"e_tests.prompt3_mqa_multiple_correct_sk:dejepis|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0": {
|
|
"acc": 0.2,
|
|
"acc_stderr": 0.04020151261036846,
|
|
"em_relaxed": 0.22,
|
|
"em_relaxed_stderr": 0.04163331998932268
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:management|0": {
|
|
"acc": 0.19,
|
|
"acc_stderr": 0.03942772444036622,
|
|
"em_relaxed": 0.19,
|
|
"em_relaxed_stderr": 0.03942772444036622
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0": {
|
|
"acc": 0.2206896551724138,
|
|
"acc_stderr": 0.03455930201924812,
|
|
"em_relaxed": 0.2413793103448276,
|
|
"em_relaxed_stderr": 0.03565998174135302
|
|
},
|
|
"sklep.prompt1_nli_3_classes_sk:nli|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:security_studies|0": {
|
|
"acc": 0.18468468468468469,
|
|
"acc_stderr": 0.026102500004017645,
|
|
"em_relaxed": 0.1981981981981982,
|
|
"em_relaxed_stderr": 0.026815582053277203
|
|
},
|
|
"sklep.prompt3_nli_binary_sk:rte|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0": {
|
|
"acc": 0.18624161073825504,
|
|
"acc_stderr": 0.015959806298600995,
|
|
"em_relaxed": 0.1912751677852349,
|
|
"em_relaxed_stderr": 0.01612394106868719
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:nutrition|0": {
|
|
"acc": 0.22772277227722773,
|
|
"acc_stderr": 0.024131614233378947,
|
|
"em_relaxed": 0.264026402640264,
|
|
"em_relaxed_stderr": 0.0253659612714007
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0": {
|
|
"acc": 0.19,
|
|
"acc_stderr": 0.039427724440366234,
|
|
"em_relaxed": 0.19,
|
|
"em_relaxed_stderr": 0.039427724440366234
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0": {
|
|
"acc": 0.21,
|
|
"acc_stderr": 0.04093601807403326,
|
|
"em_relaxed": 0.22,
|
|
"em_relaxed_stderr": 0.04163331998932268
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0": {
|
|
"acc": 0.24646983311938384,
|
|
"acc_stderr": 0.015450495390660532,
|
|
"em_relaxed": 0.24775353016688062,
|
|
"em_relaxed_stderr": 0.015477478391132707
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0": {
|
|
"acc": 0.31666666666666665,
|
|
"acc_stderr": 0.034768900963930385,
|
|
"em_relaxed": 0.31666666666666665,
|
|
"em_relaxed_stderr": 0.034768900963930385
|
|
},
|
|
"arc.prompt2_mqa_single_correct_sk|0": {
|
|
"acc": 0.23353293413173654,
|
|
"acc_stderr": 0.012379394294094546,
|
|
"em_relaxed": 0.23353293413173654,
|
|
"em_relaxed_stderr": 0.012379394294094546
|
|
},
|
|
"slovak_tests.prompt2_mqa_single_correct_sk:monitor9|0": {
|
|
"acc": 0.14746543778801843,
|
|
"acc_stderr": 0.024125384514879104,
|
|
"em_relaxed": 0.14746543778801843,
|
|
"em_relaxed_stderr": 0.024125384514879104
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0": {
|
|
"acc": 0.22,
|
|
"acc_stderr": 0.04163331998932269,
|
|
"em_relaxed": 0.22,
|
|
"em_relaxed_stderr": 0.04163331998932269
|
|
},
|
|
"slovak_tests.prompt2_mqa_single_correct_sk:vyssie_studium|0": {
|
|
"acc": 0.11428571428571428,
|
|
"acc_stderr": 0.031197955091381097,
|
|
"em_relaxed": 0.11428571428571428,
|
|
"em_relaxed_stderr": 0.031197955091381097
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0": {
|
|
"acc": 0.2359249329758713,
|
|
"acc_stderr": 0.02201321444385349,
|
|
"em_relaxed": 0.2359249329758713,
|
|
"em_relaxed_stderr": 0.02201321444385349
|
|
},
|
|
"e_tests.prompt2_mqa_multiple_correct_sk:obcianska-nauka|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"e_tests.prompt2_mqa_multiple_correct_sk:biologia|0": {
|
|
"acc": 0.017857142857142856,
|
|
"acc_stderr": 0.017857142857142867
|
|
},
|
|
"toxic_sk.prompt2_toxicity_sk|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"e_tests.prompt2_mqa_single_correct_sk:fyzika|0": {
|
|
"acc": 0.13636363636363635,
|
|
"acc_stderr": 0.07488677009526491,
|
|
"em_relaxed": 0.22727272727272727,
|
|
"em_relaxed_stderr": 0.09144861547306321
|
|
},
|
|
"senti_sk.prompt1_sentiment_binary_sk|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0": {
|
|
"acc": 0.25288831835686776,
|
|
"acc_stderr": 0.015583584105316876,
|
|
"em_relaxed": 0.25417201540436457,
|
|
"em_relaxed_stderr": 0.01560965861721762
|
|
},
|
|
"slovak_tests.prompt2_qa_sk:monitor9|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0": {
|
|
"acc": 0.23049645390070922,
|
|
"acc_stderr": 0.025123739226872395,
|
|
"em_relaxed": 0.23404255319148937,
|
|
"em_relaxed_stderr": 0.025257861359432386
|
|
},
|
|
"e_tests.prompt1_mqa_single_correct_sk:informatika|0": {
|
|
"acc": 0.264,
|
|
"acc_stderr": 0.03958494337416042,
|
|
"em_relaxed": 0.272,
|
|
"em_relaxed_stderr": 0.0399612715739433
|
|
},
|
|
"e_tests.prompt3_mqa_multiple_correct_sk:slovensky-jazyk|0": {
|
|
"acc": 0.02142857142857143,
|
|
"acc_stderr": 0.012282463299882668
|
|
},
|
|
"e_tests.prompt3_mqa_multiple_correct_sk:geografia|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0": {
|
|
"acc": 0.2349799732977303,
|
|
"acc_stderr": 0.015502474593163047,
|
|
"em_relaxed": 0.2349799732977303,
|
|
"em_relaxed_stderr": 0.015502474593163047
|
|
},
|
|
"e_tests.prompt1_qa_sk:chemia|0": {
|
|
"qem": 0.018518518518518517,
|
|
"qem_stderr": 0.018518518518518517,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.00026455026455026457,
|
|
"rouge1_stderr": 0.00026455026455026484,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.00026455026455026457,
|
|
"rougeL_stderr": 0.00026455026455026484
|
|
},
|
|
"scio.prompt2_mqa_single_correct_sk:aj|0": {
|
|
"acc": 0.6,
|
|
"acc_stderr": 0.16329931618554522,
|
|
"em_relaxed": 0.6,
|
|
"em_relaxed_stderr": 0.16329931618554522
|
|
},
|
|
"e_tests.prompt2_mqa_multiple_correct_sk:informatika|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0": {
|
|
"acc": 0.19638242894056848,
|
|
"acc_stderr": 0.020220056896337255,
|
|
"em_relaxed": 0.20155038759689922,
|
|
"em_relaxed_stderr": 0.020418410166350903
|
|
},
|
|
"slovak_tests.prompt2_mqa_single_correct_sk:maturita|0": {
|
|
"acc": 0.15176151761517614,
|
|
"acc_stderr": 0.018703199518943774,
|
|
"em_relaxed": 0.15447154471544716,
|
|
"em_relaxed_stderr": 0.01883928643967625
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0": {
|
|
"acc": 0.37373737373737376,
|
|
"acc_stderr": 0.04887069039502487,
|
|
"em_relaxed": 0.3838383838383838,
|
|
"em_relaxed_stderr": 0.04912566964083466
|
|
},
|
|
"e_tests.prompt2_qa_sk:biologia|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0": {
|
|
"acc": 0.18,
|
|
"acc_stderr": 0.03861229196653697,
|
|
"em_relaxed": 0.18,
|
|
"em_relaxed_stderr": 0.03861229196653697
|
|
},
|
|
"e_tests.prompt1_qa_sk:informatika|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.00824175824175824,
|
|
"qf1_stderr": 0.006643857376866051,
|
|
"rouge1": 0.00033664858861620243,
|
|
"rouge1_stderr": 0.00019799367371799002,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.00033664858861620243,
|
|
"rougeL_stderr": 0.00019799367371799002
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0": {
|
|
"acc": 0.22784810126582278,
|
|
"acc_stderr": 0.03347528264060925,
|
|
"em_relaxed": 0.23417721518987342,
|
|
"em_relaxed_stderr": 0.03379766010403987
|
|
},
|
|
"sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"e_tests.prompt2_mqa_single_correct_sk:informatika|0": {
|
|
"acc": 0.224,
|
|
"acc_stderr": 0.037440705810827146,
|
|
"em_relaxed": 0.232,
|
|
"em_relaxed_stderr": 0.03790650637827826
|
|
},
|
|
"e_tests.prompt2_qa_sk:obcianska-nauka|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:management|0": {
|
|
"acc": 0.21,
|
|
"acc_stderr": 0.040936018074033256,
|
|
"em_relaxed": 0.21,
|
|
"em_relaxed_stderr": 0.040936018074033256
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:international_law|0": {
|
|
"acc": 0.23728813559322035,
|
|
"acc_stderr": 0.03933012549934385,
|
|
"em_relaxed": 0.23728813559322035,
|
|
"em_relaxed_stderr": 0.03933012549934385
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0": {
|
|
"acc": 0.2488479262672811,
|
|
"acc_stderr": 0.029417385564063876,
|
|
"em_relaxed": 0.2626728110599078,
|
|
"em_relaxed_stderr": 0.029944068647646538
|
|
},
|
|
"e_tests.prompt3_qa_sk:fyzika|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0014626760356334343,
|
|
"rouge1_stderr": 0.0010249453223271077,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0014626760356334343,
|
|
"rougeL_stderr": 0.0010249453223271077
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:marketing|0": {
|
|
"acc": 0.13247863247863248,
|
|
"acc_stderr": 0.022209309073165606,
|
|
"em_relaxed": 0.13247863247863248,
|
|
"em_relaxed_stderr": 0.022209309073165606
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0": {
|
|
"acc": 0.19172932330827067,
|
|
"acc_stderr": 0.024182419147367247,
|
|
"em_relaxed": 0.19924812030075187,
|
|
"em_relaxed_stderr": 0.02453709604824174
|
|
},
|
|
"slovak_tests.prompt1_qa_sk:maturita|0": {
|
|
"qem": 0.01327433628318584,
|
|
"qem_stderr": 0.007629803632081149,
|
|
"qf1": 0.00454104547024901,
|
|
"qf1_stderr": 0.0026612932202592188,
|
|
"rouge1": 0.001362195307144739,
|
|
"rouge1_stderr": 0.0004933551702641079,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0012709627553702657,
|
|
"rougeL_stderr": 0.0004501834684090662
|
|
},
|
|
"e_tests.prompt1_mqa_multiple_correct_sk:dejepis|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"senti_sk.prompt3_sentiment_3_class_sk|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0": {
|
|
"acc": 0.1919191919191919,
|
|
"acc_stderr": 0.028057791672989017,
|
|
"em_relaxed": 0.1919191919191919,
|
|
"em_relaxed_stderr": 0.028057791672989017
|
|
},
|
|
"e_tests.prompt1_qa_sk:dejepis|0": {
|
|
"qem": 0.0,
|
|
"qem_stderr": 0.0,
|
|
"qf1": 0.0,
|
|
"qf1_stderr": 0.0,
|
|
"rouge1": 0.0,
|
|
"rouge1_stderr": 0.0,
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rougeL": 0.0,
|
|
"rougeL_stderr": 0.0
|
|
},
|
|
"sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"e_tests.prompt1_mqa_multiple_correct_sk:biologia|0": {
|
|
"acc": 0.017857142857142856,
|
|
"acc_stderr": 0.017857142857142867
|
|
},
|
|
"senti_sk.prompt1_sentiment_3_class_sk|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_biology|0": {
|
|
"acc": 0.2323943661971831,
|
|
"acc_stderr": 0.03556905428023101,
|
|
"em_relaxed": 0.2323943661971831,
|
|
"em_relaxed_stderr": 0.03556905428023101
|
|
},
|
|
"slohry.prompt2_identity_form_sk:identity_form_mid|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt1_declension_sk:declension_mid|0": {
|
|
"acc": 0.011976047904191617,
|
|
"acc_stderr": 0.005960989845848317,
|
|
"em_norm_relaxed": 0.011976047904191617,
|
|
"em_norm_relaxed_stderr": 0.005960989845848317
|
|
},
|
|
"slohry.prompt1_declension_sk:declension_low|0": {
|
|
"acc": 0.0029940119760479044,
|
|
"acc_stderr": 0.0029940119760478844,
|
|
"em_norm_relaxed": 0.0029940119760479044,
|
|
"em_norm_relaxed_stderr": 0.0029940119760478844
|
|
},
|
|
"analogies_sk.prompt1_analogy_sk|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"slovak_trivia.prompt3_mqa_single_correct_sk|0": {
|
|
"acc": 0.22096317280453256,
|
|
"acc_stderr": 0.022114015575065166,
|
|
"em_relaxed": 0.2237960339943343,
|
|
"em_relaxed_stderr": 0.02221481955471835
|
|
},
|
|
"slohry.prompt3_identity_form_sk:identity_form_low|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt3_gradation_sk:gradation_high|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt1_inflection_sk:inflection_high|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt1_gradation_sk:gradation_low|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"chiby.prompt2_chiby_sk|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt3_inflection_sk:inflection_high|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slovak_trivia.prompt2_mqa_single_correct_sk|0": {
|
|
"acc": 0.1501416430594901,
|
|
"acc_stderr": 0.01903936786327736,
|
|
"em_relaxed": 0.1558073654390935,
|
|
"em_relaxed_stderr": 0.019330515671326005
|
|
},
|
|
"chiby.prompt3_chiby_sk|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt1_gradation_sk:gradation_mid|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt1_declension_sk:declension_high|0": {
|
|
"acc": 0.017964071856287425,
|
|
"acc_stderr": 0.0072785348116838205,
|
|
"em_norm_relaxed": 0.017964071856287425,
|
|
"em_norm_relaxed_stderr": 0.0072785348116838205
|
|
},
|
|
"slohry.prompt2_inflection_sk:inflection_low|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt2_declension_sk:declension_low|0": {
|
|
"acc": 0.0029940119760479044,
|
|
"acc_stderr": 0.0029940119760478896,
|
|
"em_norm_relaxed": 0.0029940119760479044,
|
|
"em_norm_relaxed_stderr": 0.0029940119760478896
|
|
},
|
|
"slohry.prompt3_gradation_sk:gradation_low|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt2_gradation_sk:gradation_mid|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"chiby.prompt1_chiby_sk|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt3_declension_sk:declension_low|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt3_identity_form_sk:identity_form_high|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt1_identity_form_sk:identity_form_mid|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt3_declension_sk:declension_high|0": {
|
|
"acc": 0.011976047904191617,
|
|
"acc_stderr": 0.005960989845848319,
|
|
"em_norm_relaxed": 0.011976047904191617,
|
|
"em_norm_relaxed_stderr": 0.005960989845848319
|
|
},
|
|
"slohry.prompt3_declension_sk:declension_mid|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt1_inflection_sk:inflection_low|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slovak_trivia.prompt1_mqa_single_correct_sk|0": {
|
|
"acc": 0.22662889518413598,
|
|
"acc_stderr": 0.02231414649207437,
|
|
"em_relaxed": 0.23229461756373937,
|
|
"em_relaxed_stderr": 0.022508447165267422
|
|
},
|
|
"slohry.prompt1_identity_form_sk:identity_form_low|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt2_declension_sk:declension_high|0": {
|
|
"acc": 0.020958083832335328,
|
|
"acc_stderr": 0.007849722658097238,
|
|
"em_norm_relaxed": 0.020958083832335328,
|
|
"em_norm_relaxed_stderr": 0.007849722658097238
|
|
},
|
|
"slohry.prompt2_inflection_sk:inflection_high|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt2_declension_sk:declension_mid|0": {
|
|
"acc": 0.014970059880239521,
|
|
"acc_stderr": 0.006654483730236385,
|
|
"em_norm_relaxed": 0.014970059880239521,
|
|
"em_norm_relaxed_stderr": 0.006654483730236385
|
|
},
|
|
"analogies_sk.prompt2_analogy_sk|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"slohry.prompt1_gradation_sk:gradation_high|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt3_gradation_sk:gradation_mid|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt3_inflection_sk:inflection_mid|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt3_identity_form_sk:identity_form_mid|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt2_gradation_sk:gradation_low|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"analogies_sk.prompt3_analogy_sk|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"slohry.prompt1_identity_form_sk:identity_form_high|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt2_identity_form_sk:identity_form_low|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt2_inflection_sk:inflection_mid|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt3_inflection_sk:inflection_low|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt2_gradation_sk:gradation_high|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt1_inflection_sk:inflection_mid|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"slohry.prompt2_identity_form_sk:identity_form_high|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0
|
|
},
|
|
"e_tests_mqa_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.21755858691850297,
|
|
"acc": 0.15008726003490402,
|
|
"em_relaxed_stderr": 0.03505820050399378,
|
|
"acc_stderr": 0.027287029108301525,
|
|
"em_relaxed__prompts": {
|
|
"1": 0.23819517313746066,
|
|
"2": 0.2088142707240294,
|
|
"3": 0.20566631689401887
|
|
},
|
|
"acc__prompts": {
|
|
"1": 0.16454749439042632,
|
|
"2": 0.14360508601346297,
|
|
"3": 0.14210919970082272
|
|
}
|
|
},
|
|
"e_tests_mqa_AVERAGE|0": {
|
|
"em_relaxed": 0.21755858691850297,
|
|
"em_relaxed_stderr": 0.03505820050399378,
|
|
"acc": 0.15008726003490402,
|
|
"acc_stderr": 0.027287029108301525
|
|
},
|
|
"mmlu_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.22816822334507222,
|
|
"acc": 0.2224518960853366,
|
|
"em_relaxed_stderr": 0.025975113405158696,
|
|
"acc_stderr": 0.02571374453987217,
|
|
"em_relaxed__prompts": {
|
|
"1": 0.24628693921298422,
|
|
"2": 0.21719491655182974,
|
|
"3": 0.22102281427040268
|
|
},
|
|
"acc__prompts": {
|
|
"1": 0.2399326290001531,
|
|
"2": 0.21137651201959884,
|
|
"3": 0.21604654723625785
|
|
}
|
|
},
|
|
"mmlu_AVERAGE|0": {
|
|
"em_relaxed": 0.22816822334507222,
|
|
"em_relaxed_stderr": 0.025975113405158696,
|
|
"acc": 0.2224518960853366,
|
|
"acc_stderr": 0.02571374453987217
|
|
},
|
|
"scio_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.24911297852474323,
|
|
"acc": 0.238468720821662,
|
|
"em_relaxed_stderr": 0.020694508860887174,
|
|
"acc_stderr": 0.0204222231602884,
|
|
"em_relaxed__prompts": {
|
|
"1": 0.261624649859944,
|
|
"3": 0.23865546218487396,
|
|
"2": 0.24705882352941178
|
|
},
|
|
"acc__prompts": {
|
|
"1": 0.2504201680672269,
|
|
"3": 0.2280112044817927,
|
|
"2": 0.23697478991596638
|
|
}
|
|
},
|
|
"scio_AVERAGE|0": {
|
|
"em_relaxed": 0.24911297852474323,
|
|
"em_relaxed_stderr": 0.020694508860887174,
|
|
"acc": 0.238468720821662,
|
|
"acc_stderr": 0.0204222231602884
|
|
},
|
|
"slovak_tests_qa_SK_AVERAGE|0": {
|
|
"rouge2": 0.0,
|
|
"rouge1": 0.0011809129165421361,
|
|
"rougeL": 0.0011384377140699951,
|
|
"qf1": 0.003009101538513303,
|
|
"qem": 0.012605042016806723,
|
|
"rouge2_stderr": 0.0,
|
|
"rouge1_stderr": 0.0007262749376038493,
|
|
"rougeL_stderr": 0.0006990122730613884,
|
|
"qf1_stderr": 0.0019761043274910683,
|
|
"qem_stderr": 0.0072451076506316785,
|
|
"rouge2__prompts": {
|
|
"3": 0.0,
|
|
"2": 0.0,
|
|
"1": 0.0
|
|
},
|
|
"rouge1__prompts": {
|
|
"3": 0.001309304483429979,
|
|
"2": 0.0008896015463691078,
|
|
"1": 0.001343832719827322
|
|
},
|
|
"rougeL__prompts": {
|
|
"3": 0.001309304483429979,
|
|
"2": 0.0008488085301334873,
|
|
"1": 0.0012572001286465196
|
|
},
|
|
"qf1__prompts": {
|
|
"3": 0.0021008403361344537,
|
|
"2": 0.0026143790849673205,
|
|
"1": 0.004312085194438136
|
|
},
|
|
"qem__prompts": {
|
|
"3": 0.012605042016806723,
|
|
"2": 0.012605042016806723,
|
|
"1": 0.012605042016806723
|
|
}
|
|
},
|
|
"slovak_tests_qa_AVERAGE|0": {
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rouge1": 0.0011809129165421361,
|
|
"rouge1_stderr": 0.0007262749376038493,
|
|
"rougeL": 0.0011384377140699951,
|
|
"rougeL_stderr": 0.0006990122730613884,
|
|
"qf1": 0.003009101538513303,
|
|
"qf1_stderr": 0.0019761043274910683,
|
|
"qem": 0.012605042016806723,
|
|
"qem_stderr": 0.0072451076506316785
|
|
},
|
|
"slovak_tests_mqa_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.17543859649122806,
|
|
"acc": 0.17389966143428748,
|
|
"em_relaxed_stderr": 0.02232972578404266,
|
|
"acc_stderr": 0.02225738292492629,
|
|
"em_relaxed__prompts": {
|
|
"1": 0.20498614958448755,
|
|
"3": 0.17820867959372114,
|
|
"2": 0.14312096029547552
|
|
},
|
|
"acc__prompts": {
|
|
"1": 0.20221606648199447,
|
|
"3": 0.17820867959372114,
|
|
"2": 0.14127423822714683
|
|
}
|
|
},
|
|
"slovak_tests_mqa_AVERAGE|0": {
|
|
"em_relaxed": 0.17543859649122806,
|
|
"em_relaxed_stderr": 0.02232972578404266,
|
|
"acc": 0.17389966143428748,
|
|
"acc_stderr": 0.02225738292492629
|
|
},
|
|
"movie_review_SK_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.0,
|
|
"acc": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed__prompts": {
|
|
"2": 0.0,
|
|
"3": 0.0,
|
|
"1": 0.0
|
|
},
|
|
"acc__prompts": {
|
|
"2": 0.0,
|
|
"3": 0.0,
|
|
"1": 0.0
|
|
}
|
|
},
|
|
"movie_review_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0,
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"e_tests_qa_SK_AVERAGE|0": {
|
|
"rouge2": 0.0,
|
|
"rouge1": 0.00029664081444656757,
|
|
"rougeL": 0.00029664081444656757,
|
|
"qf1": 0.0026099107814683436,
|
|
"qem": 0.002257336343115124,
|
|
"rouge2_stderr": 0.0,
|
|
"rouge1_stderr": 0.0001935712320593315,
|
|
"rougeL_stderr": 0.0001935712320593315,
|
|
"qf1_stderr": 0.002326519969477271,
|
|
"qem_stderr": 0.002257336343115124,
|
|
"rouge2__prompts": {
|
|
"1": 0.0,
|
|
"2": 0.0,
|
|
"3": 0.0
|
|
},
|
|
"rouge1__prompts": {
|
|
"1": 0.00011697710542177868,
|
|
"2": 0.0002733713466381956,
|
|
"3": 0.0004995739912797284
|
|
},
|
|
"rougeL__prompts": {
|
|
"1": 0.00011697710542177868,
|
|
"2": 0.0002733713466381956,
|
|
"3": 0.0004995739912797284
|
|
},
|
|
"qf1__prompts": {
|
|
"1": 0.0025798129635601417,
|
|
"2": 0.00290228958400516,
|
|
"3": 0.002347629796839729
|
|
},
|
|
"qem__prompts": {
|
|
"1": 0.002257336343115124,
|
|
"2": 0.002257336343115124,
|
|
"3": 0.002257336343115124
|
|
}
|
|
},
|
|
"e_tests_qa_AVERAGE|0": {
|
|
"rouge2": 0.0,
|
|
"rouge2_stderr": 0.0,
|
|
"rouge1": 0.00029664081444656757,
|
|
"rouge1_stderr": 0.0001935712320593315,
|
|
"rougeL": 0.00029664081444656757,
|
|
"rougeL_stderr": 0.0001935712320593315,
|
|
"qf1": 0.0026099107814683436,
|
|
"qf1_stderr": 0.002326519969477271,
|
|
"qem": 0.002257336343115124,
|
|
"qem_stderr": 0.002257336343115124
|
|
},
|
|
"sklep_nli_SK_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.0,
|
|
"acc": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed__prompts": {
|
|
"3": 0.0,
|
|
"2": 0.0,
|
|
"1": 0.0
|
|
},
|
|
"acc__prompts": {
|
|
"3": 0.0,
|
|
"2": 0.0,
|
|
"1": 0.0
|
|
}
|
|
},
|
|
"sklep_nli_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0,
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"sklep_sentiment_analysis_SK_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.0,
|
|
"acc": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed__prompts": {
|
|
"2": 0.0,
|
|
"1": 0.0,
|
|
"3": 0.0
|
|
},
|
|
"acc__prompts": {
|
|
"2": 0.0,
|
|
"1": 0.0,
|
|
"3": 0.0
|
|
}
|
|
},
|
|
"sklep_sentiment_analysis_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0,
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"arc_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.24493869404049046,
|
|
"acc": 0.24493869404049046,
|
|
"em_relaxed_stderr": 0.012565550837641016,
|
|
"acc_stderr": 0.012565550837641016,
|
|
"em_relaxed__prompts": {
|
|
"1": 0.2728828058169376,
|
|
"3": 0.22840034217279725,
|
|
"2": 0.23353293413173654
|
|
},
|
|
"acc__prompts": {
|
|
"1": 0.2728828058169376,
|
|
"3": 0.22840034217279725,
|
|
"2": 0.23353293413173654
|
|
}
|
|
},
|
|
"arc_AVERAGE|0": {
|
|
"em_relaxed": 0.24493869404049046,
|
|
"em_relaxed_stderr": 0.012565550837641016,
|
|
"acc": 0.24493869404049046,
|
|
"acc_stderr": 0.012565550837641016
|
|
},
|
|
"belebele_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.2611111111111111,
|
|
"acc": 0.2537037037037037,
|
|
"em_relaxed_stderr": 0.014646428295956382,
|
|
"acc_stderr": 0.014509896842640889,
|
|
"em_relaxed__prompts": {
|
|
"2": 0.25666666666666665,
|
|
"3": 0.2544444444444444,
|
|
"1": 0.2722222222222222
|
|
},
|
|
"acc__prompts": {
|
|
"2": 0.2511111111111111,
|
|
"3": 0.24666666666666667,
|
|
"1": 0.2633333333333333
|
|
}
|
|
},
|
|
"belebele_AVERAGE|0": {
|
|
"em_relaxed": 0.2611111111111111,
|
|
"em_relaxed_stderr": 0.014646428295956382,
|
|
"acc": 0.2537037037037037,
|
|
"acc_stderr": 0.014509896842640889
|
|
},
|
|
"senti_sk_3class_SK_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.0,
|
|
"acc": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed__prompts": {
|
|
"2": 0.0,
|
|
"3": 0.0,
|
|
"1": 0.0
|
|
},
|
|
"acc__prompts": {
|
|
"2": 0.0,
|
|
"3": 0.0,
|
|
"1": 0.0
|
|
}
|
|
},
|
|
"senti_sk_3class_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0,
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"sklep_hate_speech_SK_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.0,
|
|
"acc": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed__prompts": {
|
|
"2": 0.0,
|
|
"1": 0.0,
|
|
"3": 0.0
|
|
},
|
|
"acc__prompts": {
|
|
"2": 0.0,
|
|
"1": 0.0,
|
|
"3": 0.0
|
|
}
|
|
},
|
|
"sklep_hate_speech_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0,
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"slovakcopa_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.05333333333333334,
|
|
"acc": 0.05333333333333334,
|
|
"em_relaxed_stderr": 0.007876087666852084,
|
|
"acc_stderr": 0.007876087666852084,
|
|
"em_relaxed__prompts": {
|
|
"3": 0.144,
|
|
"2": 0.006,
|
|
"1": 0.01
|
|
},
|
|
"acc__prompts": {
|
|
"3": 0.144,
|
|
"2": 0.006,
|
|
"1": 0.01
|
|
}
|
|
},
|
|
"slovakcopa_AVERAGE|0": {
|
|
"em_relaxed": 0.05333333333333334,
|
|
"em_relaxed_stderr": 0.007876087666852084,
|
|
"acc": 0.05333333333333334,
|
|
"acc_stderr": 0.007876087666852084
|
|
},
|
|
"globalmmlu_lite_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.21333333333333335,
|
|
"acc": 0.21083333333333334,
|
|
"em_relaxed_stderr": 0.020468272436489743,
|
|
"acc_stderr": 0.020379605193633,
|
|
"em_relaxed__prompts": {
|
|
"1": 0.2425,
|
|
"3": 0.205,
|
|
"2": 0.1925
|
|
},
|
|
"acc__prompts": {
|
|
"1": 0.24,
|
|
"3": 0.2025,
|
|
"2": 0.19
|
|
}
|
|
},
|
|
"globalmmlu_lite_AVERAGE|0": {
|
|
"em_relaxed": 0.21333333333333335,
|
|
"em_relaxed_stderr": 0.020468272436489743,
|
|
"acc": 0.21083333333333334,
|
|
"acc_stderr": 0.020379605193633
|
|
},
|
|
"sklep_qa_SK_AVERAGE|0": {
|
|
"qf1": 0.0070376260418807705,
|
|
"qem": 0.00027827054854081884,
|
|
"qf1_stderr": 0.0003824771262616922,
|
|
"qem_stderr": 0.0001679351079586058,
|
|
"qf1__prompts": {
|
|
"3": 0.006093522810123945,
|
|
"2": 0.006743145308527306,
|
|
"1": 0.008276210006991061
|
|
},
|
|
"qem__prompts": {
|
|
"3": 0.0002087029114056141,
|
|
"2": 0.0002087029114056141,
|
|
"1": 0.0004174058228112282
|
|
}
|
|
},
|
|
"sklep_qa_AVERAGE|0": {
|
|
"qf1": 0.0070376260418807705,
|
|
"qf1_stderr": 0.0003824771262616922,
|
|
"qem": 0.00027827054854081884,
|
|
"qem_stderr": 0.0001679351079586058
|
|
},
|
|
"financial_exams_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.08995502248875563,
|
|
"acc": 0.0887056471764118,
|
|
"em_relaxed_stderr": 0.007689948855002123,
|
|
"acc_stderr": 0.007626104510560348,
|
|
"em_relaxed__prompts": {
|
|
"1": 0.05697151424287856,
|
|
"2": 0.07946026986506746,
|
|
"3": 0.13343328335832083
|
|
},
|
|
"acc__prompts": {
|
|
"1": 0.053973013493253376,
|
|
"2": 0.07871064467766117,
|
|
"3": 0.13343328335832083
|
|
}
|
|
},
|
|
"financial_exams_AVERAGE|0": {
|
|
"em_relaxed": 0.08995502248875563,
|
|
"em_relaxed_stderr": 0.007689948855002123,
|
|
"acc": 0.0887056471764118,
|
|
"acc_stderr": 0.007626104510560348
|
|
},
|
|
"ifeval_sk_SK_AVERAGE|0": {
|
|
"prompt_level_strict_acc": 0.0782122905027933,
|
|
"prompt_level_loose_acc": 0.0782122905027933,
|
|
"inst_level_strict_acc": 0.17312348668280872,
|
|
"inst_level_loose_acc": 0.17312348668280872,
|
|
"prompt_level_strict_acc_stderr": 0.011597665111537078,
|
|
"prompt_level_loose_acc_stderr": 0.011597665111537078,
|
|
"inst_level_strict_acc_stderr": 0.0004210039776560394,
|
|
"inst_level_loose_acc_stderr": 0.0004210039776560394
|
|
},
|
|
"ifeval_sk_AVERAGE|0": {
|
|
"prompt_level_strict_acc": 0.0782122905027933,
|
|
"prompt_level_strict_acc_stderr": 0.011597665111537078,
|
|
"prompt_level_loose_acc": 0.0782122905027933,
|
|
"prompt_level_loose_acc_stderr": 0.011597665111537078,
|
|
"inst_level_strict_acc": 0.17312348668280872,
|
|
"inst_level_strict_acc_stderr": 0.0004210039776560394,
|
|
"inst_level_loose_acc": 0.17312348668280872,
|
|
"inst_level_loose_acc_stderr": 0.0004210039776560394
|
|
},
|
|
"toxic_sk_SK_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.0,
|
|
"acc": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed__prompts": {
|
|
"1": 0.0,
|
|
"3": 0.0,
|
|
"2": 0.0
|
|
},
|
|
"acc__prompts": {
|
|
"1": 0.0,
|
|
"3": 0.0,
|
|
"2": 0.0
|
|
}
|
|
},
|
|
"toxic_sk_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0,
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"driving_test_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.1407513076557299,
|
|
"acc": 0.13742272943414172,
|
|
"em_relaxed_stderr": 0.013052324431028241,
|
|
"acc_stderr": 0.012921192285229888,
|
|
"em_relaxed__prompts": {
|
|
"2": 0.10271041369472182,
|
|
"1": 0.16833095577746077,
|
|
"3": 0.15121255349500715
|
|
},
|
|
"acc__prompts": {
|
|
"2": 0.09985734664764621,
|
|
"1": 0.16405135520684735,
|
|
"3": 0.14835948644793154
|
|
}
|
|
},
|
|
"driving_test_AVERAGE|0": {
|
|
"em_relaxed": 0.1407513076557299,
|
|
"em_relaxed_stderr": 0.013052324431028241,
|
|
"acc": 0.13742272943414172,
|
|
"acc_stderr": 0.012921192285229888
|
|
},
|
|
"sklep_rte_SK_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.0,
|
|
"acc": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed__prompts": {
|
|
"1": 0.0,
|
|
"2": 0.0,
|
|
"3": 0.0
|
|
},
|
|
"acc__prompts": {
|
|
"1": 0.0,
|
|
"2": 0.0,
|
|
"3": 0.0
|
|
}
|
|
},
|
|
"sklep_rte_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0,
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"multi_wiki_qa_SK_AVERAGE|0": {
|
|
"qf1": 0.005263913929238519,
|
|
"qem": 0.0005336535254486025,
|
|
"qf1_stderr": 0.0005228307644937593,
|
|
"qem_stderr": 0.00032536056243172176,
|
|
"qf1__prompts": {
|
|
"3": 0.005765729879562343,
|
|
"2": 0.005417769336679904,
|
|
"1": 0.004608242571473309
|
|
},
|
|
"qem__prompts": {
|
|
"3": 0.0006003602161296778,
|
|
"2": 0.0006003602161296778,
|
|
"1": 0.00040024014408645187
|
|
}
|
|
},
|
|
"multi_wiki_qa_AVERAGE|0": {
|
|
"qf1": 0.005263913929238519,
|
|
"qf1_stderr": 0.0005228307644937593,
|
|
"qem": 0.0005336535254486025,
|
|
"qem_stderr": 0.00032536056243172176
|
|
},
|
|
"truthful_qa_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.09682947729220223,
|
|
"acc": 0.06469580119965725,
|
|
"em_relaxed_stderr": 0.010525690171591523,
|
|
"acc_stderr": 0.008619652748939895,
|
|
"em_relaxed__prompts": {
|
|
"2": 0.08354755784061697,
|
|
"1": 0.07840616966580977,
|
|
"3": 0.12853470437017994
|
|
},
|
|
"acc__prompts": {
|
|
"2": 0.04884318766066838,
|
|
"1": 0.042416452442159386,
|
|
"3": 0.10282776349614396
|
|
}
|
|
},
|
|
"truthful_qa_AVERAGE|0": {
|
|
"em_relaxed": 0.09682947729220223,
|
|
"em_relaxed_stderr": 0.010525690171591523,
|
|
"acc": 0.06469580119965725,
|
|
"acc_stderr": 0.008619652748939895
|
|
},
|
|
"senti_sk_binary_SK_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.0,
|
|
"acc": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed__prompts": {
|
|
"3": 0.0,
|
|
"2": 0.0,
|
|
"1": 0.0
|
|
},
|
|
"acc__prompts": {
|
|
"3": 0.0,
|
|
"2": 0.0,
|
|
"1": 0.0
|
|
}
|
|
},
|
|
"senti_sk_binary_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0,
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"skj1_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.61,
|
|
"acc": 0.5966666666666667,
|
|
"em_relaxed_stderr": 0.04721666475945084,
|
|
"acc_stderr": 0.04715805991356215,
|
|
"em_relaxed__prompts": {
|
|
"1": 0.44,
|
|
"3": 0.75,
|
|
"2": 0.64
|
|
},
|
|
"acc__prompts": {
|
|
"1": 0.41,
|
|
"3": 0.75,
|
|
"2": 0.63
|
|
}
|
|
},
|
|
"skj1_AVERAGE|0": {
|
|
"em_relaxed": 0.61,
|
|
"em_relaxed_stderr": 0.04721666475945084,
|
|
"acc": 0.5966666666666667,
|
|
"acc_stderr": 0.04715805991356215
|
|
},
|
|
"slohry_SK_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.002331002331002331,
|
|
"acc": 0.002331002331002331,
|
|
"em_norm_relaxed_stderr": 0.0011036777204322752,
|
|
"acc_stderr": 0.0011036777204322752,
|
|
"em_norm_relaxed__prompts": {
|
|
"2": 0.003246753246753247,
|
|
"1": 0.0027472527472527475,
|
|
"3": 0.000999000999000999
|
|
},
|
|
"acc__prompts": {
|
|
"2": 0.003246753246753247,
|
|
"1": 0.0027472527472527475,
|
|
"3": 0.000999000999000999
|
|
}
|
|
},
|
|
"slohry_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.002331002331002331,
|
|
"em_norm_relaxed_stderr": 0.0011036777204322752,
|
|
"acc": 0.002331002331002331,
|
|
"acc_stderr": 0.0011036777204322752
|
|
},
|
|
"analogies_sk_SK_AVERAGE|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"acc__prompts": {
|
|
"1": 0.0,
|
|
"2": 0.0,
|
|
"3": 0.0
|
|
}
|
|
},
|
|
"analogies_sk_AVERAGE|0": {
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"slovak_trivia_SK_AVERAGE|0": {
|
|
"em_relaxed": 0.20396600566572237,
|
|
"acc": 0.19924457034938622,
|
|
"em_relaxed_stderr": 0.021351260797103925,
|
|
"acc_stderr": 0.021155843310138964,
|
|
"em_relaxed__prompts": {
|
|
"3": 0.2237960339943343,
|
|
"2": 0.1558073654390935,
|
|
"1": 0.23229461756373937
|
|
},
|
|
"acc__prompts": {
|
|
"3": 0.22096317280453256,
|
|
"2": 0.1501416430594901,
|
|
"1": 0.22662889518413598
|
|
}
|
|
},
|
|
"slovak_trivia_AVERAGE|0": {
|
|
"em_relaxed": 0.20396600566572237,
|
|
"em_relaxed_stderr": 0.021351260797103925,
|
|
"acc": 0.19924457034938622,
|
|
"acc_stderr": 0.021155843310138964
|
|
},
|
|
"chiby_SK_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.0,
|
|
"acc": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0,
|
|
"acc_stderr": 0.0,
|
|
"em_norm_relaxed__prompts": {
|
|
"2": 0.0,
|
|
"3": 0.0,
|
|
"1": 0.0
|
|
},
|
|
"acc__prompts": {
|
|
"2": 0.0,
|
|
"3": 0.0,
|
|
"1": 0.0
|
|
}
|
|
},
|
|
"chiby_AVERAGE|0": {
|
|
"em_norm_relaxed": 0.0,
|
|
"em_norm_relaxed_stderr": 0.0,
|
|
"acc": 0.0,
|
|
"acc_stderr": 0.0
|
|
},
|
|
"SK_AVERAGE|0": {
|
|
"em_relaxed": 0.21419205155386342,
|
|
"acc": 0.10986595916434654,
|
|
"rouge2": 0.0,
|
|
"rouge1": 0.0007387768654943519,
|
|
"rougeL": 0.0007175392642582814,
|
|
"qf1": 0.004480138072775234,
|
|
"qem": 0.003918575608477817,
|
|
"em_norm_relaxed": 0.0002331002331002331,
|
|
"prompt_level_strict_acc": 0.0782122905027933,
|
|
"prompt_level_loose_acc": 0.0782122905027933,
|
|
"inst_level_strict_acc": 0.17312348668280872,
|
|
"inst_level_loose_acc": 0.17312348668280872,
|
|
"em_relaxed_stderr": 0.006300951163940428,
|
|
"acc_stderr": 0.003263889036835735,
|
|
"rouge2_stderr": 0.0,
|
|
"rouge1_stderr": 0.00037581415183320424,
|
|
"rougeL_stderr": 0.0003626596130572774,
|
|
"qf1_stderr": 0.0007801171483309406,
|
|
"qem_stderr": 0.0018993619641089511,
|
|
"em_norm_relaxed_stderr": 0.00011036777204322752,
|
|
"prompt_level_strict_acc_stderr": 0.011597665111537078,
|
|
"prompt_level_loose_acc_stderr": 0.011597665111537078,
|
|
"inst_level_strict_acc_stderr": 0.0004210039776560394,
|
|
"inst_level_loose_acc_stderr": 0.0004210039776560394
|
|
},
|
|
"AVERAGE|0": {
|
|
"em_relaxed": 0.18120805585001873,
|
|
"acc": 0.09273256151262048,
|
|
"rouge2": 0.0,
|
|
"rouge1": 0.0007387768654943519,
|
|
"rougeL": 0.0007175392642582814,
|
|
"qf1": 0.004480138072775234,
|
|
"qem": 0.003918575608477817,
|
|
"em_norm_relaxed": 0.000259000259000259,
|
|
"prompt_level_strict_acc": 0.0782122905027933,
|
|
"prompt_level_loose_acc": 0.0782122905027933,
|
|
"inst_level_strict_acc": 0.17312348668280872,
|
|
"inst_level_loose_acc": 0.17312348668280872,
|
|
"em_relaxed_stderr": 0.005577871787987613,
|
|
"acc_stderr": 0.002843082460505095,
|
|
"rouge2_stderr": 0.0,
|
|
"rouge1_stderr": 0.00037581415183320424,
|
|
"rougeL_stderr": 0.0003626596130572774,
|
|
"qf1_stderr": 0.0007801171483309406,
|
|
"qem_stderr": 0.0018993619641089511,
|
|
"em_norm_relaxed_stderr": 0.00012263085782580836,
|
|
"prompt_level_strict_acc_stderr": 0.011597665111537078,
|
|
"prompt_level_loose_acc_stderr": 0.011597665111537078,
|
|
"inst_level_strict_acc_stderr": 0.0004210039776560394,
|
|
"inst_level_loose_acc_stderr": 0.0004210039776560394
|
|
}
|
|
}
|
|
} |