Jakub
67061cf373
Document six completed LR runs as preparation for full Aya training on Perun. Record selection of LR 0.00010 by lowest final validation loss, selected-model lm-eval and partial SK-bench results, reproducible provenance, and dataset-access/context-truncation limitations. Preserve existing 30k and Titan material.
138 lines
4.6 KiB
JSON
138 lines
4.6 KiB
JSON
{
|
|
"run_name": "sk_zero_available_without_nli_lr_0p00010_20261022_185255",
|
|
"result_file": "results_2026-10-22T21-55-04.268099.json",
|
|
"task_list_count": 222,
|
|
"configured_tasks": 222,
|
|
"scored_task_variants": 222,
|
|
"result_json_keys": 226,
|
|
"extra_result_keys_excluded_from_task_summaries": [
|
|
"all",
|
|
"mmlu.prompt1_mqa_single_correct_sk:_average|0",
|
|
"mmlu.prompt2_mqa_single_correct_sk:_average|0",
|
|
"mmlu.prompt3_mqa_single_correct_sk:_average|0"
|
|
],
|
|
"all_original_task_variants": 412,
|
|
"omitted_task_variants": 190,
|
|
"inaccessible_repositories": 12,
|
|
"inaccessible_task_variants": 187,
|
|
"additionally_excluded_nli_task_variants": 3,
|
|
"families": {
|
|
"arc": 3,
|
|
"belebele": 3,
|
|
"financial_exams": 3,
|
|
"globalmmlu-lite": 3,
|
|
"mmlu": 171,
|
|
"movie_review": 3,
|
|
"multi_wiki_qa": 3,
|
|
"senti_sk": 6,
|
|
"skj1": 3,
|
|
"sklep": 15,
|
|
"slovakcopa": 3,
|
|
"toxic_sk": 3,
|
|
"truthfulqa": 3
|
|
},
|
|
"prompt_counts": {
|
|
"1": 74,
|
|
"2": 74,
|
|
"3": 74
|
|
},
|
|
"evaluated_instances": 159426,
|
|
"instances_by_prompt": {
|
|
"1": 53142,
|
|
"2": 53142,
|
|
"3": 53142
|
|
},
|
|
"details_files_audited": 222,
|
|
"truncation_column": "model_response.truncated_tokens_count",
|
|
"truncation_values_present": 159426,
|
|
"truncation_null_values": 0,
|
|
"details_reported_samples_with_truncation": 0,
|
|
"details_reported_truncated_tokens_total": 0,
|
|
"details_reported_max_truncated_tokens_per_sample": 0,
|
|
"context_truncation_observed_in_log": true,
|
|
"actual_truncated_sample_count": null,
|
|
"console_context_truncation_warning_count": 1,
|
|
"console_context_truncation_warnings": [
|
|
"context_size + max_new_tokens=53860 which is greater than self.max_length=32768. Truncating context to 32642 tokens."
|
|
],
|
|
"truncation_interpretation": "All DETAILS counters are zero, but the console explicitly reports context truncation. Do not infer no truncation or an affected-sample count from these counters.",
|
|
"model_config": {
|
|
"model_name": "/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010",
|
|
"generation_parameters": {
|
|
"num_blocks": null,
|
|
"block_size": null,
|
|
"early_stopping": null,
|
|
"repetition_penalty": null,
|
|
"frequency_penalty": null,
|
|
"length_penalty": null,
|
|
"presence_penalty": null,
|
|
"max_new_tokens": null,
|
|
"min_new_tokens": null,
|
|
"seed": null,
|
|
"stop_tokens": null,
|
|
"temperature": 0,
|
|
"top_k": null,
|
|
"min_p": null,
|
|
"top_p": null,
|
|
"truncate_prompt": null,
|
|
"cache_implementation": null,
|
|
"response_format": null,
|
|
"skip_special_tokens": true
|
|
},
|
|
"system_prompt": null,
|
|
"cache_dir": "~/.cache/huggingface/lighteval",
|
|
"tokenizer": null,
|
|
"revision": "main",
|
|
"dtype": "bfloat16",
|
|
"tensor_parallel_size": 1,
|
|
"data_parallel_size": 1,
|
|
"pipeline_parallel_size": 1,
|
|
"gpu_memory_utilization": 0.85,
|
|
"enable_prefix_caching": null,
|
|
"max_model_length": 32768,
|
|
"quantization": null,
|
|
"load_format": null,
|
|
"swap_space": 4,
|
|
"seed": 1234,
|
|
"trust_remote_code": false,
|
|
"add_special_tokens": true,
|
|
"multichoice_continuations_start_space": true,
|
|
"pairwise_tokenization": false,
|
|
"max_num_seqs": 128,
|
|
"max_num_batched_tokens": 2048,
|
|
"subfolder": null,
|
|
"is_async": false,
|
|
"override_chat_template": null,
|
|
"enforce_eager": true
|
|
},
|
|
"max_samples": null,
|
|
"generation_size_counts": {
|
|
"32": 216,
|
|
"126": 6
|
|
},
|
|
"original_summary_general": {
|
|
"hashes": {
|
|
"hash_examples": "316f54894dcb6f0d",
|
|
"hash_full_prompts": "ba7f9b257c923f55",
|
|
"hash_input_tokens": "8de2f010e9daf7fe",
|
|
"hash_cont_tokens": "1f06ad3e4b80f7cc"
|
|
},
|
|
"truncated": 0,
|
|
"non_truncated": 0,
|
|
"padded": 0,
|
|
"non_padded": 0
|
|
},
|
|
"pyarrow_version": "26.0.0",
|
|
"python_version": "3.12.14 (main, Aug 25 2026, 14:01:42) [MSC v.1944 64 bit (AMD64)]",
|
|
"aggregation": "Descriptive metric-specific task macro means and sample-weighted means; no official full-SK-bench overall score.",
|
|
"limitations": [
|
|
"Partial available zero-shot subset: 222 of 412 task variants.",
|
|
"Only the validation-selected LR=0.00010 was benchmarked.",
|
|
"Sample counts are evaluation instances; prompt variants reuse underlying questions.",
|
|
"One console warning reports context truncation; all DETAILS counters are zero, so the recorded counters do not reliably enumerate affected inputs.",
|
|
"The audit cannot recover the number of affected tasks/samples or quantify output generation-size termination.",
|
|
"All-zero summary counters include non_truncated=0 and cannot prove absence of truncation.",
|
|
"Raw DETAILS and generation-cache parquets are excluded from this curated tree."
|
|
]
|
|
}
|