js886io/Diplomovka
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
67061cf373
Diplomovka/aya_300k_lr_experiments/benchmarks/lmeval/provenance/comparability_audit.json
Jakub 67061cf373 Add Aya 300k learning-rate study and benchmark results 
Document six completed LR runs as preparation for full Aya training on Perun. Record selection of LR 0.00010 by lowest final validation loss, selected-model lm-eval and partial SK-bench results, reproducible provenance, and dataset-access/context-truncation limitations. Preserve existing 30k and Titan material.
2026-10-25 00:33:24 +02:00

380 lines
13 KiB
JSON
Raw Blame History

{
"comparison_scope": "All ten Hruska Aya 100k runs; Titan excluded.",
"units": "Accuracy metrics use 0..1; SKQuad scores use 0..100. N/A remains unavailable.",
"new_result_sha256": "5774cad7440ab807be44dc5bfa82ac7cbb4c0281204fc397a10c2bbfe7b8f452",
"task_hashes_caveat": "task_hashes is empty in all eleven JSONs; equality is not a nonempty content-hash verification.",
"model_loading_difference": "Older runs load base model plus PEFT adapter; 300k loads a merged model with explicit base tokenizer. Model arguments, parameter counts and model SHA therefore differ.",
"new_n_samples": {
"truthfulqa_sk_mc2": {
"original": 778,
"effective": 778
},
"truthfulqa_sk_mc1": {
"original": 778,
"effective": 778
},
"skquad": {
"original": 9583,
"effective": 9583
},
"sklegal": {
"original": 1334,
"effective": 1334
},
"m_mmlu_sk": {
"original": 13062,
"effective": 13062
},
"hellaswag_sk": {
"original": 9485,
"effective": 9485
},
"arc_sk": {
"original": 1169,
"effective": 1169
}
},
"comparisons": [
{
"model_id": "01_baseline_r16_a16_lr5e5_adamw",
"equal_to_300k": {
"versions": true,
"configs": true,
"task_hashes": true,
"n-samples": true,
"n-shot": true,
"git_hash": true,
"transformers_version": true,
"chat_template": true,
"chat_template_sha": true,
"system_instruction": true,
"fewshot_as_multiturn": true
},
"evaluation_config_differences": {
"model_args": {
"older_100k": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_01_baseline_r16_a16_lr5e5_adamw,dtype=bfloat16",
"new_300k": "pretrained=/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010,tokenizer=/home/schwarc/.cache/huggingface/hub/models--slovak-nlp--mistral-sk-7b/snapshots/089497ae0d72018e9591895c54f774bf0b4a83a4,dtype=bfloat16"
},
"model_num_parameters": {
"older_100k": 7283675136,
"new_300k": 7241732096
},
"model_sha": {
"older_100k": "089497ae0d72018e9591895c54f774bf0b4a83a4",
"new_300k": ""
},
"peft_sha": {
"older_100k": "",
"new_300k": null
}
}
},
{
"model_id": "02_optimizer_adamw8bit_r16_a16_lr5e5",
"equal_to_300k": {
"versions": true,
"configs": true,
"task_hashes": true,
"n-samples": true,
"n-shot": true,
"git_hash": true,
"transformers_version": true,
"chat_template": true,
"chat_template_sha": true,
"system_instruction": true,
"fewshot_as_multiturn": true
},
"evaluation_config_differences": {
"model_args": {
"older_100k": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_02_optimizer_adamw8bit_r16_a16_lr5e5,dtype=bfloat16",
"new_300k": "pretrained=/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010,tokenizer=/home/schwarc/.cache/huggingface/hub/models--slovak-nlp--mistral-sk-7b/snapshots/089497ae0d72018e9591895c54f774bf0b4a83a4,dtype=bfloat16"
},
"model_num_parameters": {
"older_100k": 7283675136,
"new_300k": 7241732096
},
"model_sha": {
"older_100k": "089497ae0d72018e9591895c54f774bf0b4a83a4",
"new_300k": ""
},
"peft_sha": {
"older_100k": "",
"new_300k": null
}
}
},
{
"model_id": "03_rank8_a16_lr5e5",
"equal_to_300k": {
"versions": true,
"configs": true,
"task_hashes": true,
"n-samples": true,
"n-shot": true,
"git_hash": true,
"transformers_version": true,
"chat_template": true,
"chat_template_sha": true,
"system_instruction": true,
"fewshot_as_multiturn": true
},
"evaluation_config_differences": {
"model_args": {
"older_100k": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_03_rank8_a16_lr5e5,dtype=bfloat16",
"new_300k": "pretrained=/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010,tokenizer=/home/schwarc/.cache/huggingface/hub/models--slovak-nlp--mistral-sk-7b/snapshots/089497ae0d72018e9591895c54f774bf0b4a83a4,dtype=bfloat16"
},
"model_num_parameters": {
"older_100k": 7262703616,
"new_300k": 7241732096
},
"model_sha": {
"older_100k": "089497ae0d72018e9591895c54f774bf0b4a83a4",
"new_300k": ""
},
"peft_sha": {
"older_100k": "",
"new_300k": null
}
}
},
{
"model_id": "04_rank32_a32_lr5e5",
"equal_to_300k": {
"versions": true,
"configs": true,
"task_hashes": true,
"n-samples": true,
"n-shot": true,
"git_hash": true,
"transformers_version": true,
"chat_template": true,
"chat_template_sha": true,
"system_instruction": true,
"fewshot_as_multiturn": true
},
"evaluation_config_differences": {
"model_args": {
"older_100k": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_04_rank32_a32_lr5e5,dtype=bfloat16",
"new_300k": "pretrained=/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010,tokenizer=/home/schwarc/.cache/huggingface/hub/models--slovak-nlp--mistral-sk-7b/snapshots/089497ae0d72018e9591895c54f774bf0b4a83a4,dtype=bfloat16"
},
"model_num_parameters": {
"older_100k": 7325618176,
"new_300k": 7241732096
},
"model_sha": {
"older_100k": "089497ae0d72018e9591895c54f774bf0b4a83a4",
"new_300k": ""
},
"peft_sha": {
"older_100k": "",
"new_300k": null
}
}
},
{
"model_id": "05_r16_alpha32_lr5e5",
"equal_to_300k": {
"versions": true,
"configs": true,
"task_hashes": true,
"n-samples": true,
"n-shot": true,
"git_hash": true,
"transformers_version": true,
"chat_template": true,
"chat_template_sha": true,
"system_instruction": true,
"fewshot_as_multiturn": true
},
"evaluation_config_differences": {
"model_args": {
"older_100k": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_05_r16_alpha32_lr5e5,dtype=bfloat16",
"new_300k": "pretrained=/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010,tokenizer=/home/schwarc/.cache/huggingface/hub/models--slovak-nlp--mistral-sk-7b/snapshots/089497ae0d72018e9591895c54f774bf0b4a83a4,dtype=bfloat16"
},
"model_num_parameters": {
"older_100k": 7283675136,
"new_300k": 7241732096
},
"model_sha": {
"older_100k": "089497ae0d72018e9591895c54f774bf0b4a83a4",
"new_300k": ""
},
"peft_sha": {
"older_100k": "",
"new_300k": null
}
}
},
{
"model_id": "06_lr1e4_r16_a16",
"equal_to_300k": {
"versions": true,
"configs": true,
"task_hashes": true,
"n-samples": true,
"n-shot": true,
"git_hash": true,
"transformers_version": true,
"chat_template": true,
"chat_template_sha": true,
"system_instruction": true,
"fewshot_as_multiturn": true
},
"evaluation_config_differences": {
"model_args": {
"older_100k": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_06_lr1e4_r16_a16,dtype=bfloat16",
"new_300k": "pretrained=/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010,tokenizer=/home/schwarc/.cache/huggingface/hub/models--slovak-nlp--mistral-sk-7b/snapshots/089497ae0d72018e9591895c54f774bf0b4a83a4,dtype=bfloat16"
},
"model_num_parameters": {
"older_100k": 7283675136,
"new_300k": 7241732096
},
"model_sha": {
"older_100k": "089497ae0d72018e9591895c54f774bf0b4a83a4",
"new_300k": ""
},
"peft_sha": {
"older_100k": "",
"new_300k": null
}
}
},
{
"model_id": "07_lr2e5_r16_a16",
"equal_to_300k": {
"versions": true,
"configs": true,
"task_hashes": true,
"n-samples": true,
"n-shot": true,
"git_hash": true,
"transformers_version": true,
"chat_template": true,
"chat_template_sha": true,
"system_instruction": true,
"fewshot_as_multiturn": true
},
"evaluation_config_differences": {
"model_args": {
"older_100k": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_07_lr2e5_r16_a16,dtype=bfloat16",
"new_300k": "pretrained=/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010,tokenizer=/home/schwarc/.cache/huggingface/hub/models--slovak-nlp--mistral-sk-7b/snapshots/089497ae0d72018e9591895c54f774bf0b4a83a4,dtype=bfloat16"
},
"model_num_parameters": {
"older_100k": 7283675136,
"new_300k": 7241732096
},
"model_sha": {
"older_100k": "089497ae0d72018e9591895c54f774bf0b4a83a4",
"new_300k": ""
},
"peft_sha": {
"older_100k": "",
"new_300k": null
}
}
},
{
"model_id": "08_attention_only_r16_a16_lr5e5",
"equal_to_300k": {
"versions": true,
"configs": true,
"task_hashes": true,
"n-samples": true,
"n-shot": true,
"git_hash": true,
"transformers_version": true,
"chat_template": true,
"chat_template_sha": true,
"system_instruction": true,
"fewshot_as_multiturn": true
},
"evaluation_config_differences": {
"model_args": {
"older_100k": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_08_attention_only_r16_a16_lr5e5,dtype=bfloat16",
"new_300k": "pretrained=/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010,tokenizer=/home/schwarc/.cache/huggingface/hub/models--slovak-nlp--mistral-sk-7b/snapshots/089497ae0d72018e9591895c54f774bf0b4a83a4,dtype=bfloat16"
},
"model_num_parameters": {
"older_100k": 7255363584,
"new_300k": 7241732096
},
"model_sha": {
"older_100k": "089497ae0d72018e9591895c54f774bf0b4a83a4",
"new_300k": ""
},
"peft_sha": {
"older_100k": "",
"new_300k": null
}
}
},
{
"model_id": "09_batch2_accum4_r16_a16_lr5e5",
"equal_to_300k": {
"versions": true,
"configs": true,
"task_hashes": true,
"n-samples": true,
"n-shot": true,
"git_hash": true,
"transformers_version": true,
"chat_template": true,
"chat_template_sha": true,
"system_instruction": true,
"fewshot_as_multiturn": true
},
"evaluation_config_differences": {
"model_args": {
"older_100k": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_09_batch2_accum4_r16_a16_lr5e5,dtype=bfloat16",
"new_300k": "pretrained=/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010,tokenizer=/home/schwarc/.cache/huggingface/hub/models--slovak-nlp--mistral-sk-7b/snapshots/089497ae0d72018e9591895c54f774bf0b4a83a4,dtype=bfloat16"
},
"model_num_parameters": {
"older_100k": 7283675136,
"new_300k": 7241732096
},
"model_sha": {
"older_100k": "089497ae0d72018e9591895c54f774bf0b4a83a4",
"new_300k": ""
},
"peft_sha": {
"older_100k": "",
"new_300k": null
}
}
},
{
"model_id": "10_r16_a32_lr1e4",
"equal_to_300k": {
"versions": true,
"configs": true,
"task_hashes": true,
"n-samples": true,
"n-shot": true,
"git_hash": true,
"transformers_version": true,
"chat_template": true,
"chat_template_sha": true,
"system_instruction": true,
"fewshot_as_multiturn": true
},
"evaluation_config_differences": {
"model_args": {
"older_100k": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_10_r16_a32_lr1e4,dtype=bfloat16",
"new_300k": "pretrained=/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010,tokenizer=/home/schwarc/.cache/huggingface/hub/models--slovak-nlp--mistral-sk-7b/snapshots/089497ae0d72018e9591895c54f774bf0b4a83a4,dtype=bfloat16"
},
"model_num_parameters": {
"older_100k": 7283675136,
"new_300k": 7241732096
},
"model_sha": {
"older_100k": "089497ae0d72018e9591895c54f774bf0b4a83a4",
"new_300k": ""
},
"peft_sha": {
"older_100k": "",
"new_300k": null
}
}
}
]
}
Reference in New Issue View Git Blame Copy Permalink