Jakub
67061cf373
Document six completed LR runs as preparation for full Aya training on Perun. Record selection of LR 0.00010 by lowest final validation loss, selected-model lm-eval and partial SK-bench results, reproducible provenance, and dataset-access/context-truncation limitations. Preserve existing 30k and Titan material.
13476 lines
443 KiB
JSON
13476 lines
443 KiB
JSON
{
|
|
"mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:miscellaneous",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"financial_exams.prompt2_completion_sk|0": {
|
|
"name": "financial_exams.prompt2_completion_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/slovak-financial-exam",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "7cdbfe635c3ed0d8b4df2ee04429d8afa651b33c",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:machine_learning|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:machine_learning",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"belebele.prompt3_mqa_single_correct_sk|0": {
|
|
"name": "belebele.prompt3_mqa_single_correct_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "facebook/belebele",
|
|
"hf_subset": "slk_Latn",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_law|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:professional_law",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:college_computer_science",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:professional_accounting",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_us_history",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_us_history",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_law|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:professional_law",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt1_qa_sk:question-answering|0": {
|
|
"name": "sklep.prompt1_qa_sk:question-answering",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "question-answering",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "exact_match_slk_prefix",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "f1_slk",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 126,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:college_chemistry",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:college_mathematics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_physics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_statistics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:professional_medicine",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"slovakcopa.prompt3_copa_sk|0": {
|
|
"name": "slovakcopa.prompt3_copa_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "NaiveNeuron/SlovakCOPA",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "5c61e4cd9343f9b4fab623bdc50bb895cc170771",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "validation",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"movie_review.prompt2_sentiment_binary_sk|0": {
|
|
"name": "movie_review.prompt2_sentiment_binary_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "mteb/slovak_movie_review_sentiment",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "29a7405aabcfd4860a51ae6f65a5650d63108f26",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt2_qa_sk:question-answering|0": {
|
|
"name": "sklep.prompt2_qa_sk:question-answering",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "question-answering",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "exact_match_slk_prefix",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "f1_slk",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 126,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_biology|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:college_biology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:nutrition|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:nutrition",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_biology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"arc.prompt3_mqa_single_correct_sk|0": {
|
|
"name": "arc.prompt3_mqa_single_correct_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "jon-tow/okapi_arc_challenge",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": [
|
|
"solve"
|
|
],
|
|
"scorer": "score",
|
|
"sample_fields": "record_to_sample",
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "56aa31aedeb55f0635cc7c3c13f7f86b276f9a4e",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:econometrics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:econometrics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:prehistory|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:prehistory",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"toxic_sk.prompt3_toxicity_sk|0": {
|
|
"name": "toxic_sk.prompt3_toxicity_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/toxic-sk",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "2fb416e18785c128e7ec934c07a14c803a88cdb3",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"belebele.prompt1_mqa_single_correct_sk|0": {
|
|
"name": "belebele.prompt1_mqa_single_correct_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "facebook/belebele",
|
|
"hf_subset": "slk_Latn",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"arc.prompt2_mqa_single_correct_sk|0": {
|
|
"name": "arc.prompt2_mqa_single_correct_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "jon-tow/okapi_arc_challenge",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": [
|
|
"solve"
|
|
],
|
|
"scorer": "score",
|
|
"sample_fields": "record_to_sample",
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "56aa31aedeb55f0635cc7c3c13f7f86b276f9a4e",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"multi_wiki_qa.prompt2_qa_sk|0": {
|
|
"name": "multi_wiki_qa.prompt2_qa_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "alexandrainst/multi-wiki-qa",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "exact_match_slk_prefix",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "f1_slk",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "9bdb3350487e399bd1cd873e1395290d636db5ce",
|
|
"hf_filter": "<lambda>",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"train"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 126,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:management|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:management",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:abstract_algebra",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:college_chemistry",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt3_nli_3_classes_sk:nli|0": {
|
|
"name": "sklep.prompt3_nli_3_classes_sk:nli",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "nli",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_medicine|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:college_medicine",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:business_ethics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:business_ethics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:college_computer_science",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:international_law|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:international_law",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:international_law|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:international_law",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"truthfulqa.prompt3_mqa_single_correct_sk:mc1|0": {
|
|
"name": "truthfulqa.prompt3_mqa_single_correct_sk:mc1",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "alexandrainst/m_truthfulqa",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "f0445d470f1925882b990f5f247fdcf288972f60",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"val"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:prehistory|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:prehistory",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:nutrition|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:nutrition",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:conceptual_physics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:college_computer_science",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_medicine|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:college_medicine",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_medicine|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:college_medicine",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_psychology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt2_nli_binary_sk:rte|0": {
|
|
"name": "sklep.prompt2_nli_binary_sk:rte",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "rte",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt2_hate_speech_sk:hate-speech|0": {
|
|
"name": "sklep.prompt2_hate_speech_sk:hate-speech",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "hate-speech",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:human_sexuality",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_psychology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_us_history",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:marketing|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:marketing",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:logical_fallacies",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:conceptual_physics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:virology|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:virology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:medical_genetics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:moral_scenarios",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:world_religions|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:world_religions",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"senti_sk.prompt3_sentiment_binary_sk|0": {
|
|
"name": "senti_sk.prompt3_sentiment_binary_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/senti-sk",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d",
|
|
"hf_filter": "<lambda>",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_physics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt1_nli_3_classes_sk:nli|0": {
|
|
"name": "sklep.prompt1_nli_3_classes_sk:nli",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "nli",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:moral_scenarios",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:business_ethics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:business_ethics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:public_relations|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:public_relations",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_statistics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"senti_sk.prompt2_sentiment_binary_sk|0": {
|
|
"name": "senti_sk.prompt2_sentiment_binary_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/senti-sk",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d",
|
|
"hf_filter": "<lambda>",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:medical_genetics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"belebele.prompt2_mqa_single_correct_sk|0": {
|
|
"name": "belebele.prompt2_mqa_single_correct_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "facebook/belebele",
|
|
"hf_subset": "slk_Latn",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "7899cdfa4e1e0d733fd77c848e2c273cb1d32be2",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:virology|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:virology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt2_nli_3_classes_sk:nli|0": {
|
|
"name": "sklep.prompt2_nli_3_classes_sk:nli",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "nli",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_european_history",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0": {
|
|
"name": "sklep.prompt3_sentiment_binary_sk:sentiment-analysis",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "sentiment-analysis",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:conceptual_physics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:moral_disputes",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:security_studies|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:security_studies",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"skj1.prompt1_skj1_sk|0": {
|
|
"name": "skj1.prompt1_skj1_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/SKJ1",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "e615c60b415389b629703f82d3a79810dbde6734",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train"
|
|
],
|
|
"evaluation_splits": [
|
|
"train"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_physics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:college_physics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt3_hate_speech_sk:hate-speech|0": {
|
|
"name": "sklep.prompt3_hate_speech_sk:hate-speech",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "hate-speech",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:virology|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:virology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:security_studies|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:security_studies",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"financial_exams.prompt3_completion_sk|0": {
|
|
"name": "financial_exams.prompt3_completion_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/slovak-financial-exam",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "7cdbfe635c3ed0d8b4df2ee04429d8afa651b33c",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:jurisprudence",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"senti_sk.prompt3_sentiment_3_class_sk|0": {
|
|
"name": "senti_sk.prompt3_sentiment_3_class_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/senti-sk",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:electrical_engineering",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt3_qa_sk:question-answering|0": {
|
|
"name": "sklep.prompt3_qa_sk:question-answering",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "question-answering",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "exact_match_slk_prefix",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "f1_slk",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 126,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:professional_psychology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:global_facts|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:global_facts",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:computer_security|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:computer_security",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:computer_security|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:computer_security",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_physics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:college_physics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"globalmmlu-lite.prompt2_mqa_single_correct_sk|0": {
|
|
"name": "globalmmlu-lite.prompt2_mqa_single_correct_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "CohereLabs/Global-MMLU-Lite",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "600af1bff2d447c7fc3aee22f11f607560d5fcb7",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": "1"
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_world_history",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:anatomy|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:anatomy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:security_studies|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:security_studies",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"movie_review.prompt3_sentiment_binary_sk|0": {
|
|
"name": "movie_review.prompt3_sentiment_binary_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "mteb/slovak_movie_review_sentiment",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "29a7405aabcfd4860a51ae6f65a5650d63108f26",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:college_chemistry",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:astronomy|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:astronomy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:medical_genetics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:abstract_algebra",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:electrical_engineering",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_world_history",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:philosophy|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:philosophy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:human_sexuality",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:abstract_algebra",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:world_religions|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:world_religions",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"toxic_sk.prompt1_toxicity_sk|0": {
|
|
"name": "toxic_sk.prompt1_toxicity_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/toxic-sk",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "2fb416e18785c128e7ec934c07a14c803a88cdb3",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"senti_sk.prompt1_sentiment_3_class_sk|0": {
|
|
"name": "senti_sk.prompt1_sentiment_3_class_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/senti-sk",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:college_biology|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:college_biology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:public_relations|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:public_relations",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:formal_logic|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:formal_logic",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_geography",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:prehistory|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:prehistory",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:econometrics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:econometrics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"arc.prompt1_mqa_single_correct_sk|0": {
|
|
"name": "arc.prompt1_mqa_single_correct_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "jon-tow/okapi_arc_challenge",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": [
|
|
"solve"
|
|
],
|
|
"scorer": "score",
|
|
"sample_fields": "record_to_sample",
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "56aa31aedeb55f0635cc7c3c13f7f86b276f9a4e",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:business_ethics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:business_ethics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:jurisprudence",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"truthfulqa.prompt2_mqa_single_correct_sk:mc1|0": {
|
|
"name": "truthfulqa.prompt2_mqa_single_correct_sk:mc1",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "alexandrainst/m_truthfulqa",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "f0445d470f1925882b990f5f247fdcf288972f60",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"val"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:miscellaneous",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:astronomy|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:astronomy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_european_history",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"senti_sk.prompt2_sentiment_3_class_sk|0": {
|
|
"name": "senti_sk.prompt2_sentiment_3_class_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/senti-sk",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_european_history",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:formal_logic|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:formal_logic",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0": {
|
|
"name": "sklep.prompt2_sentiment_binary_sk:sentiment-analysis",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "sentiment-analysis",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0": {
|
|
"name": "sklep.prompt1_sentiment_binary_sk:sentiment-analysis",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "sentiment-analysis",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"skj1.prompt2_skj1_sk|0": {
|
|
"name": "skj1.prompt2_skj1_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/SKJ1",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "e615c60b415389b629703f82d3a79810dbde6734",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train"
|
|
],
|
|
"evaluation_splits": [
|
|
"train"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:international_law|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:international_law",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt1_nli_binary_sk:rte|0": {
|
|
"name": "sklep.prompt1_nli_binary_sk:rte",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "rte",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:human_aging|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:human_aging",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:management|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:management",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:professional_accounting",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"truthfulqa.prompt1_mqa_single_correct_sk:mc1|0": {
|
|
"name": "truthfulqa.prompt1_mqa_single_correct_sk:mc1",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "alexandrainst/m_truthfulqa",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "f0445d470f1925882b990f5f247fdcf288972f60",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"val"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"senti_sk.prompt1_sentiment_binary_sk|0": {
|
|
"name": "senti_sk.prompt1_sentiment_binary_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/senti-sk",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "3c8a92168ff2b692c653de65c97bda573127d15d",
|
|
"hf_filter": "<lambda>",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:astronomy|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:astronomy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:human_aging|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:human_aging",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:formal_logic|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:formal_logic",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:sociology|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:sociology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:econometrics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:econometrics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:college_mathematics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:professional_psychology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:college_mathematics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"globalmmlu-lite.prompt3_mqa_single_correct_sk|0": {
|
|
"name": "globalmmlu-lite.prompt3_mqa_single_correct_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "CohereLabs/Global-MMLU-Lite",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "600af1bff2d447c7fc3aee22f11f607560d5fcb7",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": "1"
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:human_sexuality",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:computer_security|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:computer_security",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"toxic_sk.prompt2_toxicity_sk|0": {
|
|
"name": "toxic_sk.prompt2_toxicity_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/toxic-sk",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "2fb416e18785c128e7ec934c07a14c803a88cdb3",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"multi_wiki_qa.prompt3_qa_sk|0": {
|
|
"name": "multi_wiki_qa.prompt3_qa_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "alexandrainst/multi-wiki-qa",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "exact_match_slk_prefix",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "f1_slk",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "9bdb3350487e399bd1cd873e1395290d636db5ce",
|
|
"hf_filter": "<lambda>",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"train"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 126,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"financial_exams.prompt1_completion_sk|0": {
|
|
"name": "financial_exams.prompt1_completion_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "TUKE-KEMT/slovak-financial-exam",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "7cdbfe635c3ed0d8b4df2ee04429d8afa651b33c",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:philosophy|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:philosophy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:moral_disputes",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:marketing|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:marketing",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"globalmmlu-lite.prompt1_mqa_single_correct_sk|0": {
|
|
"name": "globalmmlu-lite.prompt1_mqa_single_correct_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "CohereLabs/Global-MMLU-Lite",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "600af1bff2d447c7fc3aee22f11f607560d5fcb7",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": "1"
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_law|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:professional_law",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:machine_learning|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:machine_learning",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_physics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:management|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:management",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:sociology|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:sociology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"skj1.prompt3_skj1_sk|0": {
|
|
"name": "skj1.prompt3_skj1_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/SKJ1",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "e615c60b415389b629703f82d3a79810dbde6734",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train"
|
|
],
|
|
"evaluation_splits": [
|
|
"train"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"movie_review.prompt1_sentiment_binary_sk|0": {
|
|
"name": "movie_review.prompt1_sentiment_binary_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "mteb/slovak_movie_review_sentiment",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "29a7405aabcfd4860a51ae6f65a5650d63108f26",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:anatomy|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:anatomy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:philosophy|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:philosophy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:human_aging|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:human_aging",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:professional_medicine",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:professional_psychology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:electrical_engineering",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:logical_fallacies",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt1_hate_speech_sk:hate-speech|0": {
|
|
"name": "sklep.prompt1_hate_speech_sk:hate-speech",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "hate-speech",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:miscellaneous",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:world_religions|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:world_religions",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_biology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_world_history",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_biology|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:college_biology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:global_facts|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:global_facts",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:moral_disputes",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_geography",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:moral_scenarios",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:college_physics|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:college_physics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:professional_medicine",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:logical_fallacies",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"sklep.prompt3_nli_binary_sk:rte|0": {
|
|
"name": "sklep.prompt3_nli_binary_sk:rte",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "slovak-nlp/sklep",
|
|
"hf_subset": "rte",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_norm_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative_norm",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=<lambda>, normalize_pred=<lambda>, strip_strings=True, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "10549a8c63542e6a0db4fcf5fcdc29b3e1b8c4e9",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "train",
|
|
"few_shots_select": "random_sampling_from_train",
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"slovakcopa.prompt2_copa_sk|0": {
|
|
"name": "slovakcopa.prompt2_copa_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "NaiveNeuron/SlovakCOPA",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "5c61e4cd9343f9b4fab623bdc50bb895cc170771",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "validation",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:jurisprudence",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"multi_wiki_qa.prompt1_qa_sk|0": {
|
|
"name": "multi_wiki_qa.prompt1_qa_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "alexandrainst/multi-wiki-qa",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "exact_match_slk_prefix",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=_inner_normalizer, normalize_pred=_inner_normalize_answer, strip_strings=False, type_exact_match=prefix)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "f1_slk",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "F1_score(aggregation_function=max, normalize_gold=_inner_normalize_answer, normalize_pred=_inner_normalize_answer, strip_strings=False)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "9bdb3350487e399bd1cd873e1395290d636db5ce",
|
|
"hf_filter": "<lambda>",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"train"
|
|
],
|
|
"few_shots_split": null,
|
|
"few_shots_select": null,
|
|
"generation_size": 126,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:high_school_psychology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:global_facts|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:global_facts",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:sociology|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:sociology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:high_school_biology",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt1_mqa_single_correct_sk:public_relations|0": {
|
|
"name": "mmlu.prompt1_mqa_single_correct_sk:public_relations",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_statistics",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:anatomy|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:anatomy",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:marketing|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:marketing",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:professional_accounting",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:high_school_geography",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"slovakcopa.prompt1_copa_sk|0": {
|
|
"name": "slovakcopa.prompt1_copa_sk",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "NaiveNeuron/SlovakCOPA",
|
|
"hf_subset": "default",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "5c61e4cd9343f9b4fab623bdc50bb895cc170771",
|
|
"hf_filter": null,
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "validation",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt2_mqa_single_correct_sk:nutrition|0": {
|
|
"name": "mmlu.prompt2_mqa_single_correct_sk:nutrition",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
},
|
|
"mmlu.prompt3_mqa_single_correct_sk:machine_learning|0": {
|
|
"name": "mmlu.prompt3_mqa_single_correct_sk:machine_learning",
|
|
"prompt_function": "prompt_fn",
|
|
"hf_repo": "lighteval/okapi_mmlu",
|
|
"hf_subset": "sk",
|
|
"metrics": [
|
|
{
|
|
"metric_name": "em_relaxed",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "RelaxedExtractMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=relaxed, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ExactMatches(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=True, type_exact_match=full)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
},
|
|
{
|
|
"metric_name": "em_conservative",
|
|
"higher_is_better": true,
|
|
"category": "GENERATIVE",
|
|
"sample_level_fn": "ConservativeExactMatch(aggregation_function=max, normalize_gold=None, normalize_pred=None, strip_strings=False, type_exact_match=full, fallback_mode=first_match)",
|
|
"corpus_level_fn": "mean",
|
|
"batched_compute": false
|
|
}
|
|
],
|
|
"hf_data_files": null,
|
|
"solver": null,
|
|
"scorer": null,
|
|
"sample_fields": null,
|
|
"sample_to_fewshot": null,
|
|
"filter": null,
|
|
"hf_revision": "b3e738ae24d5d96c4b425d4893f377451196cd2e",
|
|
"hf_filter": "mmlu_filter",
|
|
"hf_avail_splits": [
|
|
"train",
|
|
"validation",
|
|
"test"
|
|
],
|
|
"evaluation_splits": [
|
|
"test"
|
|
],
|
|
"few_shots_split": "dev",
|
|
"few_shots_select": null,
|
|
"generation_size": 32,
|
|
"generation_grammar": null,
|
|
"stop_sequence": [
|
|
"\n"
|
|
],
|
|
"num_samples": null,
|
|
"original_num_docs": -1,
|
|
"effective_num_docs": -1,
|
|
"must_remove_duplicate_docs": false,
|
|
"num_fewshots": 0,
|
|
"version": 0
|
|
}
|
|
}
|