{ "mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:miscellaneous", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "financial_exams.prompt2_completion_sk|0": { "name": "financial_exams.prompt2_completion_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/slovak-financial-exam", "hf_subset": "default", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "7cdbfe635c3ed0d8b4df2ee04429d8afa651b33c", "hf_filter": null, "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:machine_learning|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:machine_learning", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "belebele.prompt3_mqa_single_correct_sk|0": { "name": "belebele.prompt3_mqa_single_correct_sk", "prompt_function": "prompt_fn", "hf_repo": "facebook/belebele", "hf_subset": "slk_Latn", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2", "hf_filter": null, "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:professional_law|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:professional_law", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:college_computer_science", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:professional_accounting", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_us_history", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_us_history", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:professional_law|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:professional_law", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt1_qa_sk:question-answering|0": { "name": "sklep.prompt1_qa_sk:question-answering", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "question-answering", "metrics": [ { "metric_name": "exact_match_slk_prefix", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "f1_slk", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 126, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:college_chemistry", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:college_mathematics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_physics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_statistics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:professional_medicine", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "slovakcopa.prompt3_copa_sk|0": { "name": "slovakcopa.prompt3_copa_sk", "prompt_function": "prompt_fn", "hf_repo": "NaiveNeuron/SlovakCOPA", "hf_subset": "default", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "5c61e4cd9343f9b4fab623bdc50bb895cc170771", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "validation", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "movie_review.prompt2_sentiment_binary_sk|0": { "name": "movie_review.prompt2_sentiment_binary_sk", "prompt_function": "prompt_fn", "hf_repo": "mteb/slovak_movie_review_sentiment", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "29a7405aabcfd4860a51ae6f65a5650d63108f26", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt2_qa_sk:question-answering|0": { "name": "sklep.prompt2_qa_sk:question-answering", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "question-answering", "metrics": [ { "metric_name": "exact_match_slk_prefix", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "f1_slk", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 126, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:college_biology|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:college_biology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:nutrition|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:nutrition", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_biology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "arc.prompt3_mqa_single_correct_sk|0": { "name": "arc.prompt3_mqa_single_correct_sk", "prompt_function": "prompt_fn", "hf_repo": "jon-tow/okapi_arc_challenge", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": [ "solve" ], "scorer": "score", "sample_fields": "record_to_sample", "sample_to_fewshot": null, "filter": null, "hf_revision": "56aa31aedeb55f0635cc7c3c13f7f86b276f9a4e", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:econometrics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:econometrics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:prehistory|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:prehistory", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "toxic_sk.prompt3_toxicity_sk|0": { "name": "toxic_sk.prompt3_toxicity_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/toxic-sk", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "2fb416e18785c128e7ec934c07a14c803a88cdb3", "hf_filter": null, "hf_avail_splits": [ "train", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "belebele.prompt1_mqa_single_correct_sk|0": { "name": "belebele.prompt1_mqa_single_correct_sk", "prompt_function": "prompt_fn", "hf_repo": "facebook/belebele", "hf_subset": "slk_Latn", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2", "hf_filter": null, "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "arc.prompt2_mqa_single_correct_sk|0": { "name": "arc.prompt2_mqa_single_correct_sk", "prompt_function": "prompt_fn", "hf_repo": "jon-tow/okapi_arc_challenge", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": [ "solve" ], "scorer": "score", "sample_fields": "record_to_sample", "sample_to_fewshot": null, "filter": null, "hf_revision": "56aa31aedeb55f0635cc7c3c13f7f86b276f9a4e", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "multi_wiki_qa.prompt2_qa_sk|0": { "name": "multi_wiki_qa.prompt2_qa_sk", "prompt_function": "prompt_fn", "hf_repo": "alexandrainst/multi-wiki-qa", "hf_subset": "sk", "metrics": [ { "metric_name": "exact_match_slk_prefix", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "f1_slk", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "9bdb3350487e399bd1cd873e1395290d636db5ce", "hf_filter": "", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 126, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:management|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:management", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:abstract_algebra", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:college_chemistry", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt3_nli_3_classes_sk:nli|0": { "name": "sklep.prompt3_nli_3_classes_sk:nli", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "nli", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:college_medicine|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:college_medicine", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:business_ethics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:business_ethics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:college_computer_science", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:international_law|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:international_law", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:international_law|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:international_law", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "truthfulqa.prompt3_mqa_single_correct_sk:mc1|0": { "name": "truthfulqa.prompt3_mqa_single_correct_sk:mc1", "prompt_function": "prompt_fn", "hf_repo": "alexandrainst/m_truthfulqa", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "f0445d470f1925882b990f5f247fdcf288972f60", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "val" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:prehistory|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:prehistory", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:nutrition|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:nutrition", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:conceptual_physics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:college_computer_science", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:college_medicine|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:college_medicine", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:college_medicine|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:college_medicine", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_psychology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt2_nli_binary_sk:rte|0": { "name": "sklep.prompt2_nli_binary_sk:rte", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "rte", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt2_hate_speech_sk:hate-speech|0": { "name": "sklep.prompt2_hate_speech_sk:hate-speech", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "hate-speech", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:human_sexuality", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_psychology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_us_history", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:marketing|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:marketing", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:logical_fallacies", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:conceptual_physics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:virology|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:virology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:medical_genetics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:moral_scenarios", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:world_religions|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:world_religions", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "senti_sk.prompt3_sentiment_binary_sk|0": { "name": "senti_sk.prompt3_sentiment_binary_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/senti-sk", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d", "hf_filter": "", "hf_avail_splits": [ "train", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_physics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt1_nli_3_classes_sk:nli|0": { "name": "sklep.prompt1_nli_3_classes_sk:nli", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "nli", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:moral_scenarios", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:business_ethics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:business_ethics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:public_relations|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:public_relations", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_statistics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "senti_sk.prompt2_sentiment_binary_sk|0": { "name": "senti_sk.prompt2_sentiment_binary_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/senti-sk", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d", "hf_filter": "", "hf_avail_splits": [ "train", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:medical_genetics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "belebele.prompt2_mqa_single_correct_sk|0": { "name": "belebele.prompt2_mqa_single_correct_sk", "prompt_function": "prompt_fn", "hf_repo": "facebook/belebele", "hf_subset": "slk_Latn", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2", "hf_filter": null, "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:virology|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:virology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt2_nli_3_classes_sk:nli|0": { "name": "sklep.prompt2_nli_3_classes_sk:nli", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "nli", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_european_history", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0": { "name": "sklep.prompt3_sentiment_binary_sk:sentiment-analysis", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "sentiment-analysis", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:conceptual_physics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:moral_disputes", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:security_studies|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:security_studies", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "skj1.prompt1_skj1_sk|0": { "name": "skj1.prompt1_skj1_sk", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/SKJ1", "hf_subset": "default", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "e615c60b415389b629703f82d3a79810dbde6734", "hf_filter": null, "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:college_physics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:college_physics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt3_hate_speech_sk:hate-speech|0": { "name": "sklep.prompt3_hate_speech_sk:hate-speech", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "hate-speech", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:virology|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:virology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:security_studies|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:security_studies", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "financial_exams.prompt3_completion_sk|0": { "name": "financial_exams.prompt3_completion_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/slovak-financial-exam", "hf_subset": "default", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "7cdbfe635c3ed0d8b4df2ee04429d8afa651b33c", "hf_filter": null, "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:jurisprudence", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "senti_sk.prompt3_sentiment_3_class_sk|0": { "name": "senti_sk.prompt3_sentiment_3_class_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/senti-sk", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d", "hf_filter": null, "hf_avail_splits": [ "train", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:electrical_engineering", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt3_qa_sk:question-answering|0": { "name": "sklep.prompt3_qa_sk:question-answering", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "question-answering", "metrics": [ { "metric_name": "exact_match_slk_prefix", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "f1_slk", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 126, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:professional_psychology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:global_facts|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:global_facts", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:computer_security|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:computer_security", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:computer_security|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:computer_security", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:college_physics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:college_physics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "globalmmlu-lite.prompt2_mqa_single_correct_sk|0": { "name": "globalmmlu-lite.prompt2_mqa_single_correct_sk", "prompt_function": "prompt_fn", "hf_repo": "CohereLabs/Global-MMLU-Lite", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "600af1bff2d447c7fc3aee22f11f607560d5fcb7", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": "1" }, "mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_world_history", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:anatomy|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:anatomy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:security_studies|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:security_studies", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "movie_review.prompt3_sentiment_binary_sk|0": { "name": "movie_review.prompt3_sentiment_binary_sk", "prompt_function": "prompt_fn", "hf_repo": "mteb/slovak_movie_review_sentiment", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "29a7405aabcfd4860a51ae6f65a5650d63108f26", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:college_chemistry", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:astronomy|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:astronomy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:medical_genetics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:abstract_algebra", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:electrical_engineering", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_world_history", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:philosophy|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:philosophy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:human_sexuality", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:abstract_algebra", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:world_religions|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:world_religions", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "toxic_sk.prompt1_toxicity_sk|0": { "name": "toxic_sk.prompt1_toxicity_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/toxic-sk", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "2fb416e18785c128e7ec934c07a14c803a88cdb3", "hf_filter": null, "hf_avail_splits": [ "train", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "senti_sk.prompt1_sentiment_3_class_sk|0": { "name": "senti_sk.prompt1_sentiment_3_class_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/senti-sk", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d", "hf_filter": null, "hf_avail_splits": [ "train", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:college_biology|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:college_biology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:public_relations|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:public_relations", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:formal_logic|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:formal_logic", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_geography", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:prehistory|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:prehistory", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:econometrics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:econometrics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "arc.prompt1_mqa_single_correct_sk|0": { "name": "arc.prompt1_mqa_single_correct_sk", "prompt_function": "prompt_fn", "hf_repo": "jon-tow/okapi_arc_challenge", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": [ "solve" ], "scorer": "score", "sample_fields": "record_to_sample", "sample_to_fewshot": null, "filter": null, "hf_revision": "56aa31aedeb55f0635cc7c3c13f7f86b276f9a4e", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:business_ethics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:business_ethics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:jurisprudence", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "truthfulqa.prompt2_mqa_single_correct_sk:mc1|0": { "name": "truthfulqa.prompt2_mqa_single_correct_sk:mc1", "prompt_function": "prompt_fn", "hf_repo": "alexandrainst/m_truthfulqa", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "f0445d470f1925882b990f5f247fdcf288972f60", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "val" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:miscellaneous", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:astronomy|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:astronomy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_european_history", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "senti_sk.prompt2_sentiment_3_class_sk|0": { "name": "senti_sk.prompt2_sentiment_3_class_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/senti-sk", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d", "hf_filter": null, "hf_avail_splits": [ "train", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_european_history", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:formal_logic|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:formal_logic", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0": { "name": "sklep.prompt2_sentiment_binary_sk:sentiment-analysis", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "sentiment-analysis", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0": { "name": "sklep.prompt1_sentiment_binary_sk:sentiment-analysis", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "sentiment-analysis", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "skj1.prompt2_skj1_sk|0": { "name": "skj1.prompt2_skj1_sk", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/SKJ1", "hf_subset": "default", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "e615c60b415389b629703f82d3a79810dbde6734", "hf_filter": null, "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:international_law|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:international_law", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt1_nli_binary_sk:rte|0": { "name": "sklep.prompt1_nli_binary_sk:rte", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "rte", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:human_aging|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:human_aging", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:management|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:management", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:professional_accounting", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "truthfulqa.prompt1_mqa_single_correct_sk:mc1|0": { "name": "truthfulqa.prompt1_mqa_single_correct_sk:mc1", "prompt_function": "prompt_fn", "hf_repo": "alexandrainst/m_truthfulqa", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "f0445d470f1925882b990f5f247fdcf288972f60", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "val" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "senti_sk.prompt1_sentiment_binary_sk|0": { "name": "senti_sk.prompt1_sentiment_binary_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/senti-sk", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d", "hf_filter": "", "hf_avail_splits": [ "train", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:astronomy|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:astronomy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:human_aging|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:human_aging", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:formal_logic|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:formal_logic", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:sociology|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:sociology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:econometrics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:econometrics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:college_mathematics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:professional_psychology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:college_mathematics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "globalmmlu-lite.prompt3_mqa_single_correct_sk|0": { "name": "globalmmlu-lite.prompt3_mqa_single_correct_sk", "prompt_function": "prompt_fn", "hf_repo": "CohereLabs/Global-MMLU-Lite", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "600af1bff2d447c7fc3aee22f11f607560d5fcb7", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": "1" }, "mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:human_sexuality", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:computer_security|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:computer_security", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "toxic_sk.prompt2_toxicity_sk|0": { "name": "toxic_sk.prompt2_toxicity_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/toxic-sk", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "2fb416e18785c128e7ec934c07a14c803a88cdb3", "hf_filter": null, "hf_avail_splits": [ "train", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "multi_wiki_qa.prompt3_qa_sk|0": { "name": "multi_wiki_qa.prompt3_qa_sk", "prompt_function": "prompt_fn", "hf_repo": "alexandrainst/multi-wiki-qa", "hf_subset": "sk", "metrics": [ { "metric_name": "exact_match_slk_prefix", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "f1_slk", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "9bdb3350487e399bd1cd873e1395290d636db5ce", "hf_filter": "", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 126, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "financial_exams.prompt1_completion_sk|0": { "name": "financial_exams.prompt1_completion_sk", "prompt_function": "prompt_fn", "hf_repo": "TUKE-KEMT/slovak-financial-exam", "hf_subset": "default", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "7cdbfe635c3ed0d8b4df2ee04429d8afa651b33c", "hf_filter": null, "hf_avail_splits": [ "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:philosophy|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:philosophy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:moral_disputes", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:marketing|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:marketing", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "globalmmlu-lite.prompt1_mqa_single_correct_sk|0": { "name": "globalmmlu-lite.prompt1_mqa_single_correct_sk", "prompt_function": "prompt_fn", "hf_repo": "CohereLabs/Global-MMLU-Lite", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "600af1bff2d447c7fc3aee22f11f607560d5fcb7", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": "1" }, "mmlu.prompt1_mqa_single_correct_sk:professional_law|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:professional_law", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:machine_learning|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:machine_learning", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_physics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:management|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:management", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:sociology|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:sociology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "skj1.prompt3_skj1_sk|0": { "name": "skj1.prompt3_skj1_sk", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/SKJ1", "hf_subset": "default", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "e615c60b415389b629703f82d3a79810dbde6734", "hf_filter": null, "hf_avail_splits": [ "train" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "movie_review.prompt1_sentiment_binary_sk|0": { "name": "movie_review.prompt1_sentiment_binary_sk", "prompt_function": "prompt_fn", "hf_repo": "mteb/slovak_movie_review_sentiment", "hf_subset": "default", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "29a7405aabcfd4860a51ae6f65a5650d63108f26", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:anatomy|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:anatomy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:philosophy|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:philosophy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:human_aging|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:human_aging", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:professional_medicine", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:professional_psychology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:electrical_engineering", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:logical_fallacies", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt1_hate_speech_sk:hate-speech|0": { "name": "sklep.prompt1_hate_speech_sk:hate-speech", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "hate-speech", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:miscellaneous", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:world_religions|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:world_religions", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_biology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_world_history", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:college_biology|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:college_biology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:global_facts|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:global_facts", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:moral_disputes", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_geography", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:moral_scenarios", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:college_physics|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:college_physics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:professional_medicine", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:logical_fallacies", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "sklep.prompt3_nli_binary_sk:rte|0": { "name": "sklep.prompt3_nli_binary_sk:rte", "prompt_function": "prompt_fn", "hf_repo": "slovak-nlp/sklep", "hf_subset": "rte", "metrics": [ { "metric_name": "em_norm_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative_norm", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=, normalize_pred=, strip_strings=True, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "train", "few_shots_select": "random_sampling_from_train", "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "slovakcopa.prompt2_copa_sk|0": { "name": "slovakcopa.prompt2_copa_sk", "prompt_function": "prompt_fn", "hf_repo": "NaiveNeuron/SlovakCOPA", "hf_subset": "default", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "5c61e4cd9343f9b4fab623bdc50bb895cc170771", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "validation", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:jurisprudence", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "multi_wiki_qa.prompt1_qa_sk|0": { "name": "multi_wiki_qa.prompt1_qa_sk", "prompt_function": "prompt_fn", "hf_repo": "alexandrainst/multi-wiki-qa", "hf_subset": "sk", "metrics": [ { "metric_name": "exact_match_slk_prefix", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "f1_slk", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "9bdb3350487e399bd1cd873e1395290d636db5ce", "hf_filter": "", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "train" ], "few_shots_split": null, "few_shots_select": null, "generation_size": 126, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:high_school_psychology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:global_facts|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:global_facts", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:sociology|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:sociology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:high_school_biology", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt1_mqa_single_correct_sk:public_relations|0": { "name": "mmlu.prompt1_mqa_single_correct_sk:public_relations", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_statistics", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:anatomy|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:anatomy", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:marketing|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:marketing", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:professional_accounting", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:high_school_geography", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "slovakcopa.prompt1_copa_sk|0": { "name": "slovakcopa.prompt1_copa_sk", "prompt_function": "prompt_fn", "hf_repo": "NaiveNeuron/SlovakCOPA", "hf_subset": "default", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "5c61e4cd9343f9b4fab623bdc50bb895cc170771", "hf_filter": null, "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "validation", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt2_mqa_single_correct_sk:nutrition|0": { "name": "mmlu.prompt2_mqa_single_correct_sk:nutrition", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 }, "mmlu.prompt3_mqa_single_correct_sk:machine_learning|0": { "name": "mmlu.prompt3_mqa_single_correct_sk:machine_learning", "prompt_function": "prompt_fn", "hf_repo": "lighteval/okapi_mmlu", "hf_subset": "sk", "metrics": [ { "metric_name": "em_relaxed", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)", "corpus_level_fn": "mean", "batched_compute": false }, { "metric_name": "em_conservative", "higher_is_better": true, "category": "GENERATIVE", "sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)", "corpus_level_fn": "mean", "batched_compute": false } ], "hf_data_files": null, "solver": null, "scorer": null, "sample_fields": null, "sample_to_fewshot": null, "filter": null, "hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e", "hf_filter": "mmlu_filter", "hf_avail_splits": [ "train", "validation", "test" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": null, "generation_size": 32, "generation_grammar": null, "stop_sequence": [ "\n" ], "num_samples": null, "original_num_docs": -1, "effective_num_docs": -1, "must_remove_duplicate_docs": false, "num_fewshots": 0, "version": 0 } }