{ "comparison_scope": "All ten Hruska Aya 100k runs; Titan excluded.", "units": "Accuracy metrics use 0..1; SKQuad scores use 0..100. N/A remains unavailable.", "new_result_sha256": "5774cad7440ab807be44dc5bfa82ac7cbb4c0281204fc397a10c2bbfe7b8f452", "task_hashes_caveat": "task_hashes is empty in all eleven JSONs; equality is not a nonempty content-hash verification.", "model_loading_difference": "Older runs load base model plus PEFT adapter; 300k loads a merged model with explicit base tokenizer. Model arguments, parameter counts and model SHA therefore differ.", "new_n_samples": { "truthfulqa_sk_mc2": { "original": 778, "effective": 778 }, "truthfulqa_sk_mc1": { "original": 778, "effective": 778 }, "skquad": { "original": 9583, "effective": 9583 }, "sklegal": { "original": 1334, "effective": 1334 }, "m_mmlu_sk": { "original": 13062, "effective": 13062 }, "hellaswag_sk": { "original": 9485, "effective": 9485 }, "arc_sk": { "original": 1169, "effective": 1169 } }, "comparisons": [ { "model_id": "01_baseline_r16_a16_lr5e5_adamw", "equal_to_300k": { "versions": true, "configs": true, "task_hashes": true, "n-samples": true, "n-shot": true, "git_hash": true, "transformers_version": true, "chat_template": true, "chat_template_sha": true, "system_instruction": true, "fewshot_as_multiturn": true }, "evaluation_config_differences": { "model_args": { "older_100k": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_01_baseline_r16_a16_lr5e5_adamw,dtype=bfloat16", "new_300k": "pretrained=/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010,tokenizer=/home/schwarc/.cache/huggingface/hub/models--slovak-nlp--mistral-sk-7b/snapshots/089497ae0d72018e9591895c54f774bf0b4a83a4,dtype=bfloat16" }, "model_num_parameters": { "older_100k": 7283675136, "new_300k": 7241732096 }, "model_sha": { "older_100k": "089497ae0d72018e9591895c54f774bf0b4a83a4", "new_300k": "" }, "peft_sha": { "older_100k": "", "new_300k": null } } }, { "model_id": "02_optimizer_adamw8bit_r16_a16_lr5e5", "equal_to_300k": { "versions": true, "configs": true, "task_hashes": true, "n-samples": true, "n-shot": true, "git_hash": true, "transformers_version": true, "chat_template": true, "chat_template_sha": true, "system_instruction": true, "fewshot_as_multiturn": true }, "evaluation_config_differences": { "model_args": { "older_100k": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_02_optimizer_adamw8bit_r16_a16_lr5e5,dtype=bfloat16", "new_300k": "pretrained=/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010,tokenizer=/home/schwarc/.cache/huggingface/hub/models--slovak-nlp--mistral-sk-7b/snapshots/089497ae0d72018e9591895c54f774bf0b4a83a4,dtype=bfloat16" }, "model_num_parameters": { "older_100k": 7283675136, "new_300k": 7241732096 }, "model_sha": { "older_100k": "089497ae0d72018e9591895c54f774bf0b4a83a4", "new_300k": "" }, "peft_sha": { "older_100k": "", "new_300k": null } } }, { "model_id": "03_rank8_a16_lr5e5", "equal_to_300k": { "versions": true, "configs": true, "task_hashes": true, "n-samples": true, "n-shot": true, "git_hash": true, "transformers_version": true, "chat_template": true, "chat_template_sha": true, "system_instruction": true, "fewshot_as_multiturn": true }, "evaluation_config_differences": { "model_args": { "older_100k": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_03_rank8_a16_lr5e5,dtype=bfloat16", "new_300k": "pretrained=/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010,tokenizer=/home/schwarc/.cache/huggingface/hub/models--slovak-nlp--mistral-sk-7b/snapshots/089497ae0d72018e9591895c54f774bf0b4a83a4,dtype=bfloat16" }, "model_num_parameters": { "older_100k": 7262703616, "new_300k": 7241732096 }, "model_sha": { "older_100k": "089497ae0d72018e9591895c54f774bf0b4a83a4", "new_300k": "" }, "peft_sha": { "older_100k": "", "new_300k": null } } }, { "model_id": "04_rank32_a32_lr5e5", "equal_to_300k": { "versions": true, "configs": true, "task_hashes": true, "n-samples": true, "n-shot": true, "git_hash": true, "transformers_version": true, "chat_template": true, "chat_template_sha": true, "system_instruction": true, "fewshot_as_multiturn": true }, "evaluation_config_differences": { "model_args": { "older_100k": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_04_rank32_a32_lr5e5,dtype=bfloat16", "new_300k": "pretrained=/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010,tokenizer=/home/schwarc/.cache/huggingface/hub/models--slovak-nlp--mistral-sk-7b/snapshots/089497ae0d72018e9591895c54f774bf0b4a83a4,dtype=bfloat16" }, "model_num_parameters": { "older_100k": 7325618176, "new_300k": 7241732096 }, "model_sha": { "older_100k": "089497ae0d72018e9591895c54f774bf0b4a83a4", "new_300k": "" }, "peft_sha": { "older_100k": "", "new_300k": null } } }, { "model_id": "05_r16_alpha32_lr5e5", "equal_to_300k": { "versions": true, "configs": true, "task_hashes": true, "n-samples": true, "n-shot": true, "git_hash": true, "transformers_version": true, "chat_template": true, "chat_template_sha": true, "system_instruction": true, "fewshot_as_multiturn": true }, "evaluation_config_differences": { "model_args": { "older_100k": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_05_r16_alpha32_lr5e5,dtype=bfloat16", "new_300k": "pretrained=/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010,tokenizer=/home/schwarc/.cache/huggingface/hub/models--slovak-nlp--mistral-sk-7b/snapshots/089497ae0d72018e9591895c54f774bf0b4a83a4,dtype=bfloat16" }, "model_num_parameters": { "older_100k": 7283675136, "new_300k": 7241732096 }, "model_sha": { "older_100k": "089497ae0d72018e9591895c54f774bf0b4a83a4", "new_300k": "" }, "peft_sha": { "older_100k": "", "new_300k": null } } }, { "model_id": "06_lr1e4_r16_a16", "equal_to_300k": { "versions": true, "configs": true, "task_hashes": true, "n-samples": true, "n-shot": true, "git_hash": true, "transformers_version": true, "chat_template": true, "chat_template_sha": true, "system_instruction": true, "fewshot_as_multiturn": true }, "evaluation_config_differences": { "model_args": { "older_100k": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_06_lr1e4_r16_a16,dtype=bfloat16", "new_300k": "pretrained=/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010,tokenizer=/home/schwarc/.cache/huggingface/hub/models--slovak-nlp--mistral-sk-7b/snapshots/089497ae0d72018e9591895c54f774bf0b4a83a4,dtype=bfloat16" }, "model_num_parameters": { "older_100k": 7283675136, "new_300k": 7241732096 }, "model_sha": { "older_100k": "089497ae0d72018e9591895c54f774bf0b4a83a4", "new_300k": "" }, "peft_sha": { "older_100k": "", "new_300k": null } } }, { "model_id": "07_lr2e5_r16_a16", "equal_to_300k": { "versions": true, "configs": true, "task_hashes": true, "n-samples": true, "n-shot": true, "git_hash": true, "transformers_version": true, "chat_template": true, "chat_template_sha": true, "system_instruction": true, "fewshot_as_multiturn": true }, "evaluation_config_differences": { "model_args": { "older_100k": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_07_lr2e5_r16_a16,dtype=bfloat16", "new_300k": "pretrained=/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010,tokenizer=/home/schwarc/.cache/huggingface/hub/models--slovak-nlp--mistral-sk-7b/snapshots/089497ae0d72018e9591895c54f774bf0b4a83a4,dtype=bfloat16" }, "model_num_parameters": { "older_100k": 7283675136, "new_300k": 7241732096 }, "model_sha": { "older_100k": "089497ae0d72018e9591895c54f774bf0b4a83a4", "new_300k": "" }, "peft_sha": { "older_100k": "", "new_300k": null } } }, { "model_id": "08_attention_only_r16_a16_lr5e5", "equal_to_300k": { "versions": true, "configs": true, "task_hashes": true, "n-samples": true, "n-shot": true, "git_hash": true, "transformers_version": true, "chat_template": true, "chat_template_sha": true, "system_instruction": true, "fewshot_as_multiturn": true }, "evaluation_config_differences": { "model_args": { "older_100k": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_08_attention_only_r16_a16_lr5e5,dtype=bfloat16", "new_300k": "pretrained=/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010,tokenizer=/home/schwarc/.cache/huggingface/hub/models--slovak-nlp--mistral-sk-7b/snapshots/089497ae0d72018e9591895c54f774bf0b4a83a4,dtype=bfloat16" }, "model_num_parameters": { "older_100k": 7255363584, "new_300k": 7241732096 }, "model_sha": { "older_100k": "089497ae0d72018e9591895c54f774bf0b4a83a4", "new_300k": "" }, "peft_sha": { "older_100k": "", "new_300k": null } } }, { "model_id": "09_batch2_accum4_r16_a16_lr5e5", "equal_to_300k": { "versions": true, "configs": true, "task_hashes": true, "n-samples": true, "n-shot": true, "git_hash": true, "transformers_version": true, "chat_template": true, "chat_template_sha": true, "system_instruction": true, "fewshot_as_multiturn": true }, "evaluation_config_differences": { "model_args": { "older_100k": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_09_batch2_accum4_r16_a16_lr5e5,dtype=bfloat16", "new_300k": "pretrained=/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010,tokenizer=/home/schwarc/.cache/huggingface/hub/models--slovak-nlp--mistral-sk-7b/snapshots/089497ae0d72018e9591895c54f774bf0b4a83a4,dtype=bfloat16" }, "model_num_parameters": { "older_100k": 7283675136, "new_300k": 7241732096 }, "model_sha": { "older_100k": "089497ae0d72018e9591895c54f774bf0b4a83a4", "new_300k": "" }, "peft_sha": { "older_100k": "", "new_300k": null } } }, { "model_id": "10_r16_a32_lr1e4", "equal_to_300k": { "versions": true, "configs": true, "task_hashes": true, "n-samples": true, "n-shot": true, "git_hash": true, "transformers_version": true, "chat_template": true, "chat_template_sha": true, "system_instruction": true, "fewshot_as_multiturn": true }, "evaluation_config_differences": { "model_args": { "older_100k": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_10_r16_a32_lr1e4,dtype=bfloat16", "new_300k": "pretrained=/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010,tokenizer=/home/schwarc/.cache/huggingface/hub/models--slovak-nlp--mistral-sk-7b/snapshots/089497ae0d72018e9591895c54f774bf0b4a83a4,dtype=bfloat16" }, "model_num_parameters": { "older_100k": 7283675136, "new_300k": 7241732096 }, "model_sha": { "older_100k": "089497ae0d72018e9591895c54f774bf0b4a83a4", "new_300k": "" }, "peft_sha": { "older_100k": "", "new_300k": null } } } ] }