From aabd0b5874214b17069077819f20ddf5a929eca8 Mon Sep 17 00:00:00 2001 From: Jakub Date: Thu, 12 Nov 2026 16:51:42 +0100 Subject: [PATCH] Perun --- README.md | 7 + perun/README.md | 304 +- perun/configs/ministral.yaml | 58 + perun/configs/mistral.yaml | 2 +- perun/configs/olmo.yaml | 2 +- perun/ministral_3b_aya/.gitignore | 3 + perun/ministral_3b_aya/README.md | 78 + .../aya_test/eval_generate.yaml | 28 + .../ministral_3b_aya/aya_test/eval_loss.yaml | 25 + .../ministral_3b_aya/aya_test/evaluation.json | 9 + perun/ministral_3b_aya/aya_test/summary.json | 13 + .../configs/input_config.yaml | 53 + .../ministral_3b_aya/configs/used_config.yaml | 53 + .../requirements/training.txt | 148 + perun/ministral_3b_aya/scripts/evaluate.py | 301 ++ .../ministral_3b_aya/scripts/prepare_data.py | 356 ++ perun/ministral_3b_aya/scripts/train.py | 345 ++ .../aggregated/models__ministral-base.json | 3250 +++++++++++++++++ .../models__ministral-last-109064.json | 3250 +++++++++++++++++ perun/ministral_3b_aya/sk_bench/scores.csv | 30 + .../sk_bench/tasks_sk0_385.txt | 385 ++ .../training/data_manifest.json | 448 +++ .../training/eval_results.json | 7 + perun/ministral_3b_aya/training/run.json | 61 + .../training/train_results.json | 8 + .../training/training_eval_loss.png | Bin 0 -> 35015 bytes .../training/training_loss.png | Bin 0 -> 29717 bytes .../training/validation_loss.csv | 111 + perun/scripts/evaluate.py | 5 +- perun/scripts/prepare_data.py | 144 +- perun/scripts/train.py | 84 +- 31 files changed, 9173 insertions(+), 395 deletions(-) create mode 100644 perun/configs/ministral.yaml create mode 100644 perun/ministral_3b_aya/.gitignore create mode 100644 perun/ministral_3b_aya/README.md create mode 100644 perun/ministral_3b_aya/aya_test/eval_generate.yaml create mode 100644 perun/ministral_3b_aya/aya_test/eval_loss.yaml create mode 100644 perun/ministral_3b_aya/aya_test/evaluation.json create mode 100644 perun/ministral_3b_aya/aya_test/summary.json create mode 100644 perun/ministral_3b_aya/configs/input_config.yaml create mode 100644 perun/ministral_3b_aya/configs/used_config.yaml create mode 100644 perun/ministral_3b_aya/requirements/training.txt create mode 100644 perun/ministral_3b_aya/scripts/evaluate.py create mode 100644 perun/ministral_3b_aya/scripts/prepare_data.py create mode 100644 perun/ministral_3b_aya/scripts/train.py create mode 100644 perun/ministral_3b_aya/sk_bench/aggregated/models__ministral-base.json create mode 100644 perun/ministral_3b_aya/sk_bench/aggregated/models__ministral-last-109064.json create mode 100644 perun/ministral_3b_aya/sk_bench/scores.csv create mode 100644 perun/ministral_3b_aya/sk_bench/tasks_sk0_385.txt create mode 100644 perun/ministral_3b_aya/training/data_manifest.json create mode 100644 perun/ministral_3b_aya/training/eval_results.json create mode 100644 perun/ministral_3b_aya/training/run.json create mode 100644 perun/ministral_3b_aya/training/train_results.json create mode 100644 perun/ministral_3b_aya/training/training_eval_loss.png create mode 100644 perun/ministral_3b_aya/training/training_loss.png create mode 100644 perun/ministral_3b_aya/training/validation_loss.csv diff --git a/README.md b/README.md index 0d37d6e..18d44d0 100644 --- a/README.md +++ b/README.md @@ -975,3 +975,10 @@ Z pôvodných 412 slovenských položiek sa vynechalo 187 položiek pre 12 nedos Pri príprave sa sampler FlashInfer pokúšal kompilovať cez starší systémový CUDA toolkit. Úspešný beh používal `VLLM_USE_FLASHINFER_SAMPLER=0` a `--enforce-eager`. +# Perun – dotrénovanie Ministralu 3B na slovenskej Aya + +Model `ministral/Ministral-3b-instruct` bol dotrénovaný metódou SFT s LoRA na 3 490 041 tréningových príkladoch na štyroch NVIDIA H200. Beh dokončil jednu epochu a 109 064 krokov pri LR `0.0002`. Najnižší validačný loss bol **1,13089**. + +Na odloženej Aya množine dosiahol posledný checkpoint tokenové F1 **35,06 zo 100**. V čiastočnom slovenskom zero-shot SK-bench sa porovnali oba modely na rovnakých **385 variantoch a 29 z 33 úloh**. Overall podľa metodiky leaderboardu stúpol z **7,18 na 14,07 bodu** pri pokrytí piatich zo šiestich skupín. + +[Dokumentácia experimentu, konfigurácie a výsledky](perun/ministral_3b_aya/README.md) diff --git a/perun/README.md b/perun/README.md index 8f694d9..133c408 100644 --- a/perun/README.md +++ b/perun/README.md @@ -1,56 +1,50 @@ -# Slovenská Aya: Mistral-SK a OLMo +# Slovenská Aya na Perunovi -## Scenár - -Cieľom je inštrukčne dotrénovať Mistral-SK-7B a základný OLMo 2 7B na spoločnej slovenskej časti Aya Collection a porovnať ich adaptáciu. Použije sa **SFT s LoRA v LLaMA-Factory, zrýchlený Unslothom**. Jeden tréning využije **štyri H200 na jednom uzle Peruna cez DDP**. - -Možnosti LoRA do návrhu scenára: - -- **LoRA (Low-Rank Adaptation)** -- **QLoRA (Quantized Low-Rank Adaptation)** -- **rsLoRA (Rank-Stabilized LoRA)** -- **DoRA (Weight-Decomposed Low-Rank Adaptation)** - - -## Prehľad súborov +Prvým modelom pre ďalšiu sériu je `ministral/Ministral-3b-instruct`. Po SFT s LoRA v LLaMA-Factory sa pôvodný a dotrénovaný model porovnajú na rovnakých úlohách SK-bench. `configs/ministral.yaml` pripína rovnakú východiskovú revíziu modelu, aká sa stiahla pre baseline. Formát chatu sa odvodí z tokenizéra modelu. Konfigurácie Mistral-SK a OLMo sú v balíku ako ďalšie možnosti. | Súbor | Úloha | | --- | --- | -| `configs/mistral.yaml` | Základný tréningový config pre `slovak-nlp/mistral-sk-7b`. | -| `configs/olmo.yaml` | Základný tréningový config pre `allenai/OLMo-2-1124-7B`. | -| `scripts/prepare_data.py` | Načítanie, čistenie, deduplikácia a rozdelenie dát. | -| `scripts/train.py` | Spustenie LLaMA-Factory podľa zvoleného configu. | -| `scripts/evaluate.py` | Loss, perplexita, generovanie a slovný F1 / presná zhoda na eval alebo test množine. | +| `scripts/prepare_data.py` | Jednorazová príprava spoločného datasetu. | +| `configs/ministral.yaml` | Model, revízia, LoRA a tréningové nastavenia. | +| `scripts/train.py` | Kontrola dát a spustenie tréningu s Unslothom. | +| `scripts/evaluate.py` | Loss, perplexita, slovný F1 a presná zhoda na spoločnom splite eval alebo test. | -Balík má spolu šesť súborov. `evaluate.py` odvodí nastavenia z rovnakého modelového YAML a konkrétnu použitú konfiguráciu uloží až k výsledkom. +## Jeden pevný dataset -Poradie používania je **príprava dát → tréning → vyhodnotenie**. Príprava dát sa vykoná raz pre oba modely. Tréning sa spustí samostatne pre každý model alebo experiment a následne sa vyhodnotí vytvorený adaptér. Model a hyperparametre sa nastavujú v YAML; Python skripty vykonávajú jednotlivé kroky. +Príprava dát sa vykoná **raz**. Skript načíta slovenskú Aya, vynechá neplatné a prázdne riadky, odstráni normalizované duplicity a rozdelí dáta so seedom 42. Rovnaké zadania s rôznymi odpoveďami zostávajú v jednej množine. Eval aj test dostanú najviac 1 000 príkladov. Zvyšok patrí do train. -## Spoločný dataset do 4 096 tokenov a nastavenie H200 +V `data/aya_slovak_fixed` sa uložia **úplné zadania a referenčné odpovede**. Pri zmene modelu sa použijú tie isté súbory a ten istý manifest. Tokenizácia prebehne až v LLaMA-Factory podľa modelovej konfigurácie. Pri `cutoff_len: 4096` sa prípadné orezanie týka spracovaných tokenov pre daný beh. Dĺžky a počet orezaných príkladov sa môžu medzi modelmi líšiť. -Oba modely používajú **`cutoff_len: 4096`**. Limit zahŕňa celú SFT sekvenciu: systémový text Alpaca, inštrukciu a prípadný vstup, celú referenčnú odpoveď, EOS a koncové oddeľovače. Spúšťací skript kontroluje pôvodnú kontextovú kapacitu modelu podľa `cutoff_len`. +Manifest zaznamená zdroj, seed, počty a kontrolné súčty. `dataset_info.json` registruje `aya_train`, `aya_eval` a `aya_test`. Súbory `*.metadata.jsonl` uchovávajú pôvod príkladov v rovnakom poradí. Dočasná SQLite databáza zabezpečuje deduplikáciu a rozdelenie bez držania celého datasetu v RAM. -`prepare_data.py` si načíta oba YAML configy a ich presné tokenizéry. **Celý pár vyradí, ak prekročí limit čo i len pri jednom modeli.** Všetky ponechané príklady zapíše bez orezania. Filtrovanie prebehne pred deduplikáciou a spoločným rozdelením na train/eval/test, takže oba modely dostanú rovnaké príklady. Inštrukcia a pole `input` sa spájajú rovnako ako v Alpaca konvertore LLaMA-Factory. +## 1. Príprava dát na CPU +Balík sa prenesie do `/mnt/project/perun26012578/perun`. Zdrojový Parquet je už v `/mnt/project/perun26012578/raw/slovak_train.parquet`. Pri príprave stačí Python s PyArrow, ktorý je dostupný aj v pripravenom kontajneri SK-bench. -Nastavenie jedného tréningu: +Z prihlasovacieho uzla sa príprava odošle do `cpu_short`. Príkaz použije existujúci obraz a Python prostredie v `apphome`. -- štyri H200 cez DDP, `bf16: true`, `use_unsloth: true`, `use_unsloth_gc: true`, bez kvantizácie; -- batch 8 na GPU, akumulácia 1: **4 × 8 × 1 = efektívny batch 32**; -- eval loss s batchom 8 na GPU, záverečné generovanie po jednom príklade; -- LoRA rank 16, alpha 32, dropout 0, projekcie attention aj MLP; -- jedna epocha, AdamW, cosine scheduler, warmup 8 %, clipping gradientov 0,5; -- loss iba z odpovedí, bez packingu, dynamický padding na dĺžku aktuálnej dávky; -- validácia/checkpoint každých 1 000 optimizer krokov, výber podľa najnižšieho loss, limit checkpointov 2. +```bash +cd /mnt/project/perun26012578/perun +mkdir -p logs +export APPHOME=/mnt/home/jascke919/apphome +sbatch --job-name=aya_prepare \ + --account=perun26012578 --partition=cpu_short \ + --nodes=1 --ntasks=1 --cpus-per-task=4 --mem=24G \ + --time=08:00:00 --output=logs/aya_prepare_%j.out \ + --wrap='apptainer exec --home "$APPHOME" \ + --bind /mnt/project/perun26012578:/mnt/project/perun26012578 \ + --pwd /mnt/project/perun26012578/perun "$APPHOME/cuda.sif" \ + "$APPHOME/venv/bin/python" -u scripts/prepare_data.py \ + --local-parquet /mnt/project/perun26012578/raw/slovak_train.parquet \ + --output-dir data/aya_slovak_fixed' +``` -Spúšťač overuje, že dáta majú záznam o dĺžkovom filtrovaní pre daný model, revíziu, šablónu a limit. Kontrola pri tokenizácii tiež zastaví beh, ak by predspracovanie chcelo orezať vstup alebo odpoveď. Konfigurácia načítaného základného modelu sa uloží do `adapter/base_model_config.json`. +Po dokončení sa skutočné počty a kontrolné súčty nachádzajú v `data/aya_slovak_fixed/manifest.json`. Výstupný priečinok prípravy musí byť nový alebo prázdny. Pri vlastnom lokálnom JSONL slúži `--local-jsonl`. Bez lokálneho vstupu sa použije pripnutá revízia slovenskej Aya na Hugging Face. -LR `0.0001` pri Mistrale nadväzuje na 300k experiment; pri OLMo je východiskový kandidát. Zmenené nastavenia stále treba overiť podľa eval dát. OLMo je základný model pre vlastné SFT a jeho karta uvádza angličtinu. +## Prostredie pre tréning -## Prostredie na Perunovi - -Na spustenie sa používa kompatibilné GPU prostredie s LLaMA-Factory, Unsloth, Unsloth Zoo, PyTorch a CUDA pre H200. LLaMA-Factory používa aj `datasets`, `huggingface_hub` a `PyYAML`; generatívna evaluácia potrebuje `rouge-chinese`, `jieba` a `nltk`. Python má byť aspoň 3.11 podľa aktuálnych požiadaviek LLaMA-Factory. Dĺžkový filter pri príprave dát používa Transformers a PyYAML a načíta oba tokenizéry; váhy modelov na túto CPU úlohu nepotrebuje. +Tréning používa samostatné prostredie s LLaMA-Factory, Unslothom a CUDA pre H200. Balík obsahuje zachytené verzie knižníc. Funkčnosť konkrétnej zostavy sa overí pri prvom pilote na Perunovi. ### Požiadavky a inštalácia prostredí @@ -92,7 +86,7 @@ Na pridelenom GPU uzle sa overia importy, CUDA a BF16: python -c 'import unsloth, torch, jieba, nltk; from rouge_chinese import Rouge; assert torch.cuda.is_available(); assert torch.cuda.is_bf16_supported(); print("GPU:", torch.cuda.get_device_name(0)); print("Viditelne GPU:", torch.cuda.device_count()); print("CUDA:", torch.version.cuda)' ``` -Príprava dát, tréning aj `scripts/evaluate.py` sa potom spúšťajú v `aya_train_perun` podľa ďalších častí dokumentácie. Overené opravy generovania z Titanu sú súčasťou `scripts/train.py`. +Tréning aj `scripts/evaluate.py` sa potom spúšťajú v `aya_train_perun` podľa ďalších častí dokumentácie. Overené opravy generovania z Titanu sú súčasťou `scripts/train.py`. #### lm-evaluation-harness @@ -110,231 +104,47 @@ lm_eval --help Súbor inštaluje framework pri revízii `6d2abda4fd171e68a8789330c4149e37c1ca0bda`. Vlastné slovenské úlohy, zoznam benchmarkov a evaluačné dáta sa prenášajú samostatne; požiadavky obsahujú Python knižnice. -#### SK-bench +## 2. Pilot a tréning -Na obnovenie SK-bench sa používa uv 0.13.5 a zdrojová kópia projektu z Hrušky pri revízii `15823bbcc2d995e69441fcb7d34abb61bd8180c5`. Súbory `pyproject.toml` a `uv.lock` sa skopírujú z tohto balíka do koreňa zdrojovej kópie SK-bench. Ak je uv potrebné doplniť, nainštaluje sa cez `python -m pip install uv==0.13.5` v existujúcom Python prostredí; vlastné závislosti SK-bench sa vytvoria v jeho samostatnej `.venv`. - -Z koreňa balíka, pri zdrojovej kópii SK-bench uloženej v `../sk-bench`: +Z koreňa balíka s aktivovaným tréningovým prostredím sa overí konfigurácia a integrita spoločných dát. ```bash -cp requirements/skbench/pyproject.toml requirements/skbench/uv.lock ../sk-bench/ -cd ../sk-bench - -uv sync --locked --no-default-groups --python 3.13.17 -uv pip check --python .venv/bin/python -uv run --locked python -c 'import torch, vllm, transformers; print(torch.__version__, vllm.__version__, transformers.__version__)' +python scripts/train.py configs/ministral.yaml --dry-run ``` -Základná uzamknutá zostava zodpovedá všetkým 308 balíkom exportovaným z Hrušky. Skupiny `dev` a `hpc` sa pri tomto obnovení nepridávajú. `requirements/skbench.txt` uchováva rovnaké konkrétne verzie v pip formáte; pri používaní projektu cez uv sa vychádza z jeho `uv.lock`. +Plný tréning využíva štyri H200 na jednom uzle cez DDP. `train.py` spustí procesy sám. Slurm job preto požaduje jeden task a štyri GPU. Základná konfigurácia používa BF16, LoRA rank 16, alpha 32, batch 8 na GPU a efektívny batch 32. LR 0,0001 je východisková hodnota pre Ministral. -SK-bench používa CUDA 13; pri jeho prenose sa na pridelenom GPU uzle overí ovládač cez `nvidia-smi` a dostupnosť CUDA v PyTorchi. Pôvodný export Hrušky uvádza ovládač 580.178.04. - -Zdroje: [PyTorch CUDA zostavy](https://pytorch.org/get-started/previous-versions/#v260), [kompatibilita TorchAO](https://github.com/pytorch/ao/issues/2919), [závislosti Unsloth Zoo 2026.8.16](https://pypi.org/pypi/unsloth-zoo/2026.8.16/json), [zmeny setuptools](https://setuptools.pypa.io/en/latest/history.html#v82-0-0), [uzamknuté prostredia uv](https://docs.astral.sh/uv/concepts/projects/sync/). - -### Odosielanie úloh cez Slurm - -Priame Python príkazy sú určené pre už pridelený výpočtový uzol. Z prihlasovacieho uzla sa úlohy odosielajú cez `sbatch --wrap`. Príkazy sa zadávajú z koreňa balíka s aktivovaným Python prostredím; tento priečinok sa použije aj pri výpočte. - -Pred odoslaním sa v rovnakom termináli nastavia projektový účet a QoS. Hodnoty `NAZOV_PROJEKTU` a `PRIDELENE_QOS` sa nahradia údajmi prideleného projektu: +Prvý pilot overí načítanie modelu, jeho natívny formát chatu, priebeh lossu a uloženie adaptéra. `--pilot-steps 30` obmedzí počet krokov a pri predvolenom splite použije najviac 2 048 tréningových riadkov z pevného poradia. Celý validačný split zostane dostupný. Pilot použije vlastný výstupný priečinok. LLaMA-Factory načíta zdrojový JSONL a obmedzí rozsah pred tokenizáciou. ```bash -AYA_ACCOUNT="NAZOV_PROJEKTU" -AYA_QOS="PRIDELENE_QOS" +sbatch --job-name=ministral_pilot \ + --account=perun26012578 --partition=gpu_short \ + --nodes=1 --ntasks=1 --gres=gpu:4 --cpus-per-task=32 --mem=128G \ + --time=02:00:00 --output=logs/ministral_pilot_%j.out \ + --wrap='python -u scripts/train.py configs/ministral.yaml --run-dir runs/ministral_pilot --pilot-steps 30' ``` -Na Perunovi sa pre výpočet používa pracovný priečinok na SCRATCH. Dáta a výsledky určené na uchovanie sa po výpočte skopírujú do projektového úložiska. Parameter `--time` určuje maximálny čas úlohy. - -## 1. Príprava dát - -Príkazy sa spúšťajú z koreňa balíka s aktivovaným Python prostredím. Príprava dát používa CPU; na Perunovi sa vykonáva na výpočtovom uzle s pridelenými CPU. +Plný tréning sa spustí po kontrole pilotu a spresnení času. Pri dlhom behu sa checkpointy ukladajú každých 1 000 optimizer krokov a vyberá sa podľa najnižšieho validačného lossu. ```bash -python scripts/prepare_data.py --output-dir data/aya_slovak_4096 +sbatch --job-name=ministral_train \ + --account=perun26012578 --partition=gpu_long \ + --nodes=1 --ntasks=1 --gres=gpu:4 --cpus-per-task=32 --mem=256G \ + --time=2-00:00:00 --output=logs/ministral_train_%j.out \ + --wrap='python -u scripts/train.py configs/ministral.yaml' ``` -Odoslanie prípravy dát ako CPU úlohy: +Adaptér sa uloží do `runs/ministral/adapter`. V nadradenom priečinku zostanú `train.log`, `used_config.yaml` a `run.json` s identifikáciou datasetu. Pre iný beh sa zvolí nový `--run-dir`. Pri obnovení z checkpointu sa použije `--resume` s pôvodnými dátami a nastaveniami. + +## 3. Vyhodnotenie + +Po tréningu môže `evaluate.py` na jednej GPU vypočítať validačný loss a perplexitu a vygenerovať odpovede pre F1 a presnú zhodu. Pri generovaní sa ponechá priestor aspoň pre jeden nový token. Pri skórovaní generovaných odpovedí sa používajú úplné referencie zo spoločného JSONL. Výsledky obsahujú aj počty referencií, ktoré sa líšia od exportovaných labelov. Po výbere modelu sa môže použiť `--split test` s novým výstupným priečinkom. ```bash -sbatch --job-name=aya_prepare \ - --account="$AYA_ACCOUNT" --qos="$AYA_QOS" \ - --partition=cpu_short --nodes=1 --ntasks=1 \ - --cpus-per-task=8 --mem=64G --time=12:00:00 \ - --output=%x_%j.log \ - --wrap='python -u scripts/prepare_data.py --output-dir data/aya_slovak_4096' +python scripts/evaluate.py configs/ministral.yaml --output-dir results/ministral_eval +python scripts/evaluate.py configs/ministral.yaml --split test --output-dir results/ministral_test ``` -Tréningové úlohy sa odosielajú po úspešnom dokončení prípravy dát. +Externý SK-bench sa spúšťa zo samostatného klonu v `/mnt/project/perun26012578/sk-bench`. Na porovnanie s baseline sa použije rovnaká revízia SK-bench, rovnaké nastavenia generovania a rovnaký súbor `tasks_ministral_sk_baseline.txt` s 340 variantmi. Lokálne exportovaný model musí byť viditeľný z kontajnera. [SK-bench na Slurme](https://github.com/slovak-nlp/sk-bench/blob/main/container/slurm/README.md) -Výstupný priečinok prípravy dát musí byť nový alebo prázdny. - -### `prepare_data.py` - -Jeho vstupom sú slovenské zdrojové dáta Aya Collection a obe modelové konfigurácie. Používa CPU a tokenizéry Mistralu a OLMo. Výsledkom je jeden spoločný dátový balík pre oba modely. - -1. Načíta slovenský split `train` datasetu `CohereLabs/aya_collection_language_split` pri pevnej revízii. Zdroj číta postupne. -2. Vyradí neplatné riadky, prázdne zadania a prázdne odpovede. Polia `inputs` a `targets` prevedie na `instruction/input/output` a odstráni okrajové medzery. -3. Pomocou oboch tokenizérov zmeria celú SFT sekvenciu vrátane formátu Alpaca a odpovede. Ak príklad prekročí `cutoff_len` pri ktoromkoľvek modeli, vyradí celý pár. -4. Odstráni duplicitné normalizované páry zadania a odpovede. Rovnaké normalizované zadanie s rôznymi odpoveďami ponechá ako jednu skupinu. -5. Podľa seedu, predvolene `42`, vytvorí train/eval/test. Celá skupina zadania zostane v jednej množine. Do eval a test vyberie najviac 1 000 príkladov na každú množinu; ostatné ponechané príklady patria do train. -6. Uloží dáta vo formáte, ktorý následne načíta LLaMA-Factory. - -Príklad uloženého riadka: - -```json -{"instruction":"Aké je hlavné mesto Slovenska?","input":"","output":"Bratislava."} -``` - -Výstupy v `data/aya_slovak_4096`: - -| Súbor | Obsah a použitie | -| --- | --- | -| `train.jsonl` | Príklady na učenie LoRA parametrov. | -| `eval.jsonl` | Validačné príklady na sledovanie loss a ladenie. | -| `test.jsonl` | Odložené príklady na záverečné vyhodnotenie. | -| `dataset_info.json` | Registrácia súborov pod názvami `aya_train`, `aya_eval`, `aya_test` a mapovanie ich stĺpcov pre LLaMA-Factory. | -| `manifest.json` | Zdrojová revízia, seed, skutočné počty, nastavenia dĺžkového filtra a kontrolné súčty súborov. | -| `*.metadata.jsonl` | Pôvod a identifikácia príkladov v rovnakom poradí ako v príslušnom dátovom súbore. | -| `excluded_long.jsonl` | Identifikácia a namerané dĺžky vyradených dlhých príkladov. | - -Na deduplikáciu a rozdelenie dát používa dočasnú SQLite databázu. Po úspešnom dokončení ju odstráni; prepínač `--keep-db` ju ponechá. Počty eval a test príkladov možno nastaviť cez `--eval-size` a `--test-size`, seed cez `--seed`. Vlastný vstupný JSONL možno načítať cez `--local-jsonl`. - -Oba modely potom používajú tie isté výsledné súbory. Každý model si ich pri tréningu tokenizuje svojím tokenizérom. LLaMA-Factory používa pripravené `aya_train` a `aya_eval` bez ďalšieho automatického delenia cez `val_size`. - -## 2. Tréning - -### `train.py` - -Je to spúšťač tréningu v LLaMA-Factory. Ako vstup dostane modelový YAML a už pripravený dátový balík. - -1. Načíta model, revíziu, cesty k dátam, LoRA parametre, learning rate, batch a ostatné nastavenia z YAML. -2. Skontroluje existenciu dát, ich kontrolné súčty a zhodu dĺžkového filtra s použitým modelom, revíziou, šablónou a limitom. -3. Uloží presnú použitú konfiguráciu a záznam behu. -4. Cez `torchrun` spustí štyri procesy DDP a v každom zavolá tréning LLaMA-Factory. - -Pri aktuálnych konfiguráciách prebieha SFT s LoRA, Unslothom a BF16. Základné váhy modelu sú zmrazené a učia sa LoRA parametre. Loss sa počíta z odpovedí. LLaMA-Factory každých 1 000 optimizer krokov vykoná validáciu a uloží checkpoint; na konci načíta checkpoint s najnižším eval loss. - -Kontrola pomocou `--dry-run` sa vykonáva po príprave dát. Overí základné nastavenia a dátové súbory bez načítania váh a bez spustenia tréningu. - -Kontrola konfigurácie a pripravených dát bez spustenia tréningu: - -```bash -python scripts/train.py configs/mistral.yaml --dry-run -python scripts/train.py configs/olmo.yaml --dry-run -``` - -Každý tréning sa spúšťa samostatne v prostredí so štyrmi dostupnými GPU: - -```bash -python scripts/train.py configs/mistral.yaml -python scripts/train.py configs/olmo.yaml -``` - -Odoslanie tréningov cez Slurm na štyri H200. Nasledujúci príklad spustí modely postupne; `afterok` umožní spustenie OLMo po úspešnom dokončení Mistralu: - -```bash -AYA_MISTRAL_JOB=$(sbatch --parsable --job-name=aya_mistral \ - --account="$AYA_ACCOUNT" --qos="$AYA_QOS" \ - --partition=gpu_long --nodes=1 --ntasks=1 --gres=gpu:4 \ - --cpus-per-task=32 --mem=256G --time=2-00:00:00 \ - --output=%x_%j.log \ - --wrap='python -u scripts/train.py configs/mistral.yaml') - -sbatch --job-name=aya_olmo \ - --dependency="afterok:$AYA_MISTRAL_JOB" \ - --account="$AYA_ACCOUNT" --qos="$AYA_QOS" \ - --partition=gpu_long --nodes=1 --ntasks=1 --gres=gpu:4 \ - --cpus-per-task=32 --mem=256G --time=2-00:00:00 \ - --output=%x_%j.log \ - --wrap='python -u scripts/train.py configs/olmo.yaml' -``` - -Jedna úloha Slurmu spustí `train.py`, ktorý vytvorí štyri procesy DDP. Parametre `--run-dir` a `--resume` sa pri odosielaní uvádzajú v Python príkaze vo vnútri `--wrap`. - -Každý proces DDP má na pridelenej GPU kópiu modelu a spracúva inú časť dávky. Gradienty sa synchronizujú medzi GPU, takže štyri GPU vykonávajú jeden spoločný tréning. Pri batchi 8 na GPU a akumulácii 1 je efektívny batch **4 × 8 × 1 = 32**. Každý proces importuje Unsloth pred Transformers a LLaMA-Factory. - -Výsledkom tréningu je **LoRA adaptér**, ktorý sa používa spolu so základným modelom. Pre Mistral sa uloží do `runs/mistral/adapter`, pre OLMo do `runs/olmo/adapter`. Checkpointy sú v podpriečinkoch adaptéra. - -V nadradenom priečinku behu zostanú: - -- `train.log` – priebeh a hlásenia tréningu; -- `used_config.yaml` – presné nastavenia odovzdané LLaMA-Factory; -- `run.json` – záznam behu, efektívny batch a identifikácia použitých dát. - - -## 3. Vyhodnotenie na eval dátach - -### `evaluate.py` - -Ako vstup dostane rovnaký modelový YAML, pripravené dáta a dotrénovaný adaptér. Načíta základný model spolu s adaptérom a vykoná dve samostatné fázy na jednej GPU: - -1. **Eval loss a perplexita:** model spracuje zadania s pôvodnými referenčnými odpoveďami. Vypočíta sa strata pri predpovedaní odpovedí a z nej perplexita. -2. **Generovanie a porovnanie odpovedí:** model dostane zadania a vytvorí vlastné odpovede. Skript ich porovná s úplnými referenciami pomocou slovného F1 a normalizovanej presnej zhody. - -Predvolene sa vyhodnotí pripravená množina `eval.jsonl`: - -```bash -python scripts/evaluate.py configs/mistral.yaml --output-dir results/mistral_eval - -python scripts/evaluate.py configs/olmo.yaml --output-dir results/olmo_eval -``` - -Po úspešnom dokončení príslušného tréningu sa vyhodnotenie odošle ako úloha s jednou GPU: - -```bash -sbatch --job-name=aya_mistral_eval \ - --account="$AYA_ACCOUNT" --qos="$AYA_QOS" \ - --partition=gpu_short --nodes=1 --ntasks=1 --gres=gpu:1 \ - --cpus-per-task=8 --mem=64G --time=12:00:00 \ - --output=%x_%j.log \ - --wrap='python -u scripts/evaluate.py configs/mistral.yaml --output-dir results/mistral_eval' - -sbatch --job-name=aya_olmo_eval \ - --account="$AYA_ACCOUNT" --qos="$AYA_QOS" \ - --partition=gpu_short --nodes=1 --ntasks=1 --gres=gpu:1 \ - --cpus-per-task=8 --mem=64G --time=12:00:00 \ - --output=%x_%j.log \ - --wrap='python -u scripts/evaluate.py configs/olmo.yaml --output-dir results/olmo_eval' -``` - -Pre vyhodnotenie test množiny sa do Python príkazu vo vnútri `--wrap` pridá `--split test` a zvolí sa nový výstupný priečinok, napríklad `results/mistral_test`. - -V priečinku behu sa uchovávajú pôvodné súbory `run.json` a `used_config.yaml`. Vyhodnotenie porovná manifest pripravených dát so záznamom z tréningu; iné rozdelenie alebo iný dataset odmietne. Rovnaká kontrola platí pre eval aj test. - -Adaptér z iného behu sa určí parametrom `--adapter runs/mistral_dalsi/adapter`. Predvolene sa použije adaptér z `output_dir` v modelovom YAML. Prepínač `--dry-run` overí pripravené vstupy a zobrazí odvodené evaluačné nastavenia. - -Generovanie je greedy, s jedným lúčom a limitom najviac 256 nových tokenov. Limit možno zmeniť cez `--max-new-tokens`. Pri každom vstupe sa počet nových tokenov obmedzí aj zostávajúcim miestom do 4 096 tokenov. Celý vstup a úplná referencia zostávajú zachované; prípadné zníženie limitu sa zapíše do logu. - -Skript zapisuje výsledky do priečinka z `--output-dir`: - -| Súbor | Obsah súboru | -| --- | --- | -| `summary.json` | Hlavný súhrn: eval loss, perplexita, F1, presná zhoda a počet vyhodnotených príkladov. | -| `loss/eval_results.json` | Výsledky samostatného výpočtu eval loss v LLaMA-Factory. | -| `generation/generated_predictions.jsonl` | Vygenerované odpovede modelu. | -| `scores/per_sample.jsonl` | Predikcia, úplná referencia a skóre pre každý príklad. | -| `scores/metrics.json` | Podrobné súhrny metrík aj rozdelenie podľa typu úlohy. | -| `evaluation.json` | Zvolený split, modelový config, adaptér a stav vyhodnotenia. | - -Pri výsledkoch zostanú aj logy a konkrétne použité konfigurácie oboch evaluačných fáz. Pre vyhodnotenie možno ako vstupný config použiť aj `runs/mistral/used_config.yaml` alebo OLMo ekvivalent. - - -Eval slúži na ladenie a výber checkpointu. Po ukončení výberu nastavení sa vyhodnotí odložená testovacia množina a výsledky sa uložia do nového výstupného priečinka: - -```bash -python scripts/evaluate.py configs/mistral.yaml --split test --output-dir results/mistral_test -python scripts/evaluate.py configs/olmo.yaml --split test --output-dir results/olmo_test -``` - -Externé lm-eval a SK-bench majú vlastné spúšťanie. Tento balík obsahuje prípravu dát, tréning a vyhodnotenie na pripravených eval/test dátach. - -### Sledovanie odoslaných úloh - -```bash -squeue -u "$USER" -``` - -Výstup úlohy sa zapisuje do logu v pracovnom priečinku. V názve `%x_%j.log` sa `%x` nahradí názvom úlohy a `%j` jej číslom. Pre prípravu dát s číslom úlohy 12345: - -```bash -tail -f aya_prepare_12345.log -``` +Stav jobu sa sleduje cez `squeue -u "$USER"`. Pri chybách sa kontroluje log v `logs/`. Revízia Gitu sa v prípade potreby zistí pred odoslaním na prihlasovacom uzle. diff --git a/perun/configs/ministral.yaml b/perun/configs/ministral.yaml new file mode 100644 index 0000000..a3324f1 --- /dev/null +++ b/perun/configs/ministral.yaml @@ -0,0 +1,58 @@ +# SFT pôvodného Ministral-3b-instruct. Šablóna sa odvodí z jeho tokenizéra. +model_name_or_path: "ministral/Ministral-3b-instruct" +model_revision: "2c95908929198d6e69af8638f0dbbd9bc6b93f9e" +stage: sft +do_train: true +do_eval: true +finetuning_type: lora +use_unsloth: true +use_unsloth_gc: true +bf16: true +infer_dtype: bfloat16 +flash_attn: sdpa + +dataset_dir: data/aya_slovak_fixed +dataset: aya_train +eval_dataset: aya_eval +cutoff_len: 4096 +packing: false +train_on_prompt: false +preprocessing_num_workers: 8 +dataloader_num_workers: 2 +overwrite_cache: false + +lora_rank: 16 +lora_alpha: 32 +lora_dropout: 0.0 +lora_target: q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj + +per_device_train_batch_size: 8 +per_device_eval_batch_size: 8 +gradient_accumulation_steps: 1 +learning_rate: 0.0001 +num_train_epochs: 1.0 +lr_scheduler_type: cosine +warmup_ratio: 0.08 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 0.5 +gradient_checkpointing: true +ddp_find_unused_parameters: false + +output_dir: runs/ministral/adapter +overwrite_output_dir: false +logging_steps: 10 +eval_strategy: steps +eval_steps: 1000 +save_strategy: steps +save_steps: 1000 +save_total_limit: 2 +save_only_model: false +load_best_model_at_end: true +metric_for_best_model: loss +greater_is_better: false +predict_with_generate: false +plot_loss: true +report_to: none +seed: 42 +data_seed: 42 diff --git a/perun/configs/mistral.yaml b/perun/configs/mistral.yaml index 6b82b06..bd648af 100644 --- a/perun/configs/mistral.yaml +++ b/perun/configs/mistral.yaml @@ -12,7 +12,7 @@ bf16: true infer_dtype: bfloat16 flash_attn: sdpa -dataset_dir: data/aya_slovak_4096 +dataset_dir: data/aya_slovak_fixed dataset: aya_train eval_dataset: aya_eval cutoff_len: 4096 diff --git a/perun/configs/olmo.yaml b/perun/configs/olmo.yaml index ea3f0c5..4595dc1 100644 --- a/perun/configs/olmo.yaml +++ b/perun/configs/olmo.yaml @@ -12,7 +12,7 @@ bf16: true infer_dtype: bfloat16 flash_attn: sdpa -dataset_dir: data/aya_slovak_4096 +dataset_dir: data/aya_slovak_fixed dataset: aya_train eval_dataset: aya_eval cutoff_len: 4096 diff --git a/perun/ministral_3b_aya/.gitignore b/perun/ministral_3b_aya/.gitignore new file mode 100644 index 0000000..7b103c2 --- /dev/null +++ b/perun/ministral_3b_aya/.gitignore @@ -0,0 +1,3 @@ +/data/ +/runs/ +__pycache__/ diff --git a/perun/ministral_3b_aya/README.md b/perun/ministral_3b_aya/README.md new file mode 100644 index 0000000..598c337 --- /dev/null +++ b/perun/ministral_3b_aya/README.md @@ -0,0 +1,78 @@ +# Perun – dotrénovanie Ministralu 3B na slovenskej Aya + +Model [`ministral/Ministral-3b-instruct`](https://huggingface.co/ministral/Ministral-3b-instruct) bol dotrénovaný metódou SFT s LoRA cez LLaMA-Factory a Unsloth. Použila sa základná revízia `2c95908929198d6e69af8638f0dbbd9bc6b93f9e`. Behy prebehli na Perune. + +## Dáta a tréning + +Slovenská časť Aya Collection bola načítaná z Parquetu. Po vyradení prázdnych alebo neplatných záznamov a odstránení duplicitných párov zostalo 3 492 041 príkladov. Rovnaké normalizované prompty sa držali v jednej skupine pri delení dát. + +| Množina | Príklady | +|---|---:| +| Tréningová | 3 490 041 | +| Validačná | 1 000 | +| Testovacia | 1 000 | + +[Manifest dát](training/data_manifest.json) obsahuje počty, seed a kontrolné súčty vstupu aj pripravených súborov. Jeho SHA-256 sa zhoduje s hodnotou zaznamenanou v [metadátach tréningu](training/run.json). + +| Nastavenie jobu 148065 | Hodnota | +|---|---| +| Hardvér a presnosť | 4× NVIDIA H200 na jednom uzle, BF16 | +| Tréning | 1 epocha, 109 064 krokov | +| Efektívna dávka | 32 príkladov, 8 na GPU a akumulácia 1 | +| LoRA | Rank 16, alpha 32, dropout 0 | +| Learning rate | `0.0002`, kosínusový plán a warmup 8 % | +| Kontext | 4 096 tokenov | +| Seed | 42 pre model aj dáta | + +Tréningová slučka trvala **24 h 06 min**. Priemerný tréningový loss bol **1,33344**. Najnižší validačný loss bol **1,13089** v kroku **109 000**. Posledný checkpoint z kroku **109 064** mal validačný loss **1,13097**. Na testovanie Aya aj porovnanie v SK-bench sa použil posledný checkpoint. + +[Tréningové metriky](training/train_results.json) · [Validačný priebeh](training/validation_loss.csv) · [Graf tréningového lossu](training/training_loss.png) · [Graf validačného lossu](training/training_eval_loss.png) + +## Test na odloženej Aya množine + +Job 152588 vyhodnotil checkpoint 109 064 na 1 000 testovacích príkladoch. Pri generovaní bol limit 256 nových tokenov. Pri jednom príklade zostala po normalizácii prázdna referencia, takže tokenové F1 a presná zhoda používajú 999 hodnotiteľných odpovedí. + +| Metrika | Hodnota | +|---|---:| +| Testovací loss | 1,08998 | +| Perplexita | 2,97423 | +| Tokenové F1 (0–100) | 35,06 | +| Normalizovaná presná zhoda | 3,60 % (36 z 999) | + +[Súhrn testu](aya_test/summary.json) · [Konfigurácia výpočtu lossu](aya_test/eval_loss.yaml) · [Konfigurácia generovania](aya_test/eval_generate.yaml) + +## Čiastočné výsledky SK-bench + +Pôvodný Ministral a model so zlúčeným adaptérom z checkpointu 109 064 boli vyhodnotené na **rovnakých 385 z 412 slovenských zero-shot variantov**, ktoré pokrývajú **29 z 33 úloh**. Výsledok spája pôvodných 340 variantov (joby 145668 a 152980) a 45 doplnených variantov (joby 157393 a 157394). + +| Skupina | Pokrytie | Pôvodný | Dotrénovaný | Rozdiel | +|---|---:|---:|---:|---:| +| Knowledge & Reasoning | 10/10 | 20,57 | 23,37 | +2,80 | +| Reading & QA | 8/9 | 7,42 | 12,53 | +5,11 | +| Classification | 8/9 | 0,00 | 23,34 | +23,34 | +| Instruction | 1/1 | 7,82 | 10,24 | +2,42 | +| Grammar & Morphology | 2/2 | 0,12 | 0,88 | +0,76 | +| **Overall** | **5/6 skupín** | **7,18** | **14,07** | **+6,89** | + +Chýbajú OneRuler-SK (18 variantov), NLI-SK-annotated (3), SMESum (3) a SlovakSum (3). Skupina Generation preto nemá skóre. Overall je priemer piatich dostupných skupín s rovnakou váhou, vypočítaný postupom verejného leaderboardu. Dotrénovaný model sa zlepšil v 26 z 29 úloh. Najväčší pokles bol pri SKJ1, z 59,67 na 0,00. + +[Skóre po úlohách](sk_bench/scores.csv) · [Zoznam variantov](sk_bench/tasks_sk0_385.txt) · [Agregát pôvodného modelu](sk_bench/aggregated/models__ministral-base.json) · [Agregát dotrénovaného modelu](sk_bench/aggregated/models__ministral-last-109064.json) + +Agregované JSON-y vznikli oficiálnym skriptom z revízie SK-bench `15823bbcc2d995e69441fcb7d34abb61bd8180c5`. CSV používa hlavné metriky úloh z revízie leaderboardu `833bd28e96fd392b7064de053084dc0a6ca9a3e3`. Hodnoty sú na škále 0–100 a rozdiel je v bodoch. [Agregačný skript](https://github.com/slovak-nlp/sk-bench/blob/15823bbcc2d995e69441fcb7d34abb61bd8180c5/scripts/generate_final_results.py) · [Výpočet leaderboardu](https://huggingface.co/spaces/slovak-nlp/sk-bench-leaderboard/blob/833bd28e96fd392b7064de053084dc0a6ca9a3e3/backend/app/leaderboard.py) + +## Spustenie tréningu a testu + +Príkazy sa spúšťajú z tohto priečinka v pripravenom prostredí s [tréningovými závislosťami](requirements/training.txt). Príprava používa pôvodný slovenský Parquet. Tréningový skript spúšťa štyri procesy na uzle so štyrmi GPU. Evaluácia testovacej množiny sa spúšťa na jednej GPU. + +```bash +python scripts/prepare_data.py --local-parquet /cesta/k/slovak_train.parquet \ + --output-dir data/aya_slovak_fixed --seed 42 --eval-size 1000 --test-size 1000 + +python scripts/train.py configs/input_config.yaml --run-dir runs/ministral_aya + +python scripts/evaluate.py runs/ministral_aya/used_config.yaml \ + --adapter runs/ministral_aya/adapter/checkpoint-109064 \ + --split test --output-dir runs/ministral_aya/evaluations/aya_test --max-new-tokens 256 +``` + +[`input_config.yaml`](configs/input_config.yaml) je pôvodný vstup a `--run-dir` určuje nový výstupný priečinok. [`used_config.yaml`](configs/used_config.yaml) zaznamenáva konfiguráciu skutočne použitú pri jobe 148065 vrátane ciest na Perune. diff --git a/perun/ministral_3b_aya/aya_test/eval_generate.yaml b/perun/ministral_3b_aya/aya_test/eval_generate.yaml new file mode 100644 index 0000000..4f087c8 --- /dev/null +++ b/perun/ministral_3b_aya/aya_test/eval_generate.yaml @@ -0,0 +1,28 @@ +model_name_or_path: ministral/Ministral-3b-instruct +model_revision: 2c95908929198d6e69af8638f0dbbd9bc6b93f9e +stage: sft +finetuning_type: lora +dataset_dir: /mnt/project/perun26012578/perun/data/aya_slovak_fixed +cutoff_len: 4096 +packing: false +train_on_prompt: false +bf16: true +infer_dtype: bfloat16 +flash_attn: sdpa +use_unsloth: true +seed: 42 +data_seed: 42 +preprocessing_num_workers: 8 +dataloader_num_workers: 2 +report_to: none +adapter_name_or_path: /mnt/project/perun26012578/perun/models/ministral-3b-instruct/runs/full_aya_lr_0p000200_epoch1/adapter/checkpoint-109064 +eval_dataset: aya_test +use_unsloth_gc: false +overwrite_output_dir: false +output_dir: /mnt/project/perun26012578/perun/models/ministral-3b-instruct/runs/full_aya_lr_0p000200_epoch1/evaluations/aya_test_last_109064/generation +do_predict: true +predict_with_generate: true +per_device_eval_batch_size: 1 +do_sample: false +num_beams: 1 +max_new_tokens: 256 diff --git a/perun/ministral_3b_aya/aya_test/eval_loss.yaml b/perun/ministral_3b_aya/aya_test/eval_loss.yaml new file mode 100644 index 0000000..4e77a3f --- /dev/null +++ b/perun/ministral_3b_aya/aya_test/eval_loss.yaml @@ -0,0 +1,25 @@ +model_name_or_path: ministral/Ministral-3b-instruct +model_revision: 2c95908929198d6e69af8638f0dbbd9bc6b93f9e +stage: sft +finetuning_type: lora +dataset_dir: /mnt/project/perun26012578/perun/data/aya_slovak_fixed +cutoff_len: 4096 +packing: false +train_on_prompt: false +bf16: true +infer_dtype: bfloat16 +flash_attn: sdpa +use_unsloth: true +seed: 42 +data_seed: 42 +preprocessing_num_workers: 8 +dataloader_num_workers: 2 +report_to: none +adapter_name_or_path: /mnt/project/perun26012578/perun/models/ministral-3b-instruct/runs/full_aya_lr_0p000200_epoch1/adapter/checkpoint-109064 +eval_dataset: aya_test +use_unsloth_gc: false +overwrite_output_dir: false +output_dir: /mnt/project/perun26012578/perun/models/ministral-3b-instruct/runs/full_aya_lr_0p000200_epoch1/evaluations/aya_test_last_109064/loss +do_eval: true +predict_with_generate: false +per_device_eval_batch_size: 8 diff --git a/perun/ministral_3b_aya/aya_test/evaluation.json b/perun/ministral_3b_aya/aya_test/evaluation.json new file mode 100644 index 0000000..ed8501f --- /dev/null +++ b/perun/ministral_3b_aya/aya_test/evaluation.json @@ -0,0 +1,9 @@ +{ + "status": "complete", + "split": "test", + "adapter": "/mnt/project/perun26012578/perun/models/ministral-3b-instruct/runs/full_aya_lr_0p000200_epoch1/adapter/checkpoint-109064", + "config": "/mnt/project/perun26012578/perun/models/ministral-3b-instruct/runs/full_aya_lr_0p000200_epoch1/used_config.yaml", + "gpus": 1, + "backend": "Unsloth", + "precision": "bf16" +} diff --git a/perun/ministral_3b_aya/aya_test/summary.json b/perun/ministral_3b_aya/aya_test/summary.json new file mode 100644 index 0000000..c6368ec --- /dev/null +++ b/perun/ministral_3b_aya/aya_test/summary.json @@ -0,0 +1,13 @@ +{ + "model": "ministral/Ministral-3b-instruct", + "split": "test", + "eval_loss": 1.089984655380249, + "perplexity": 2.974228433808542, + "token_f1": 35.06239252655205, + "normalized_exact_match": 3.6036036036036037, + "examples": 1000, + "metric_scale": "0..100", + "max_new_tokens": 256, + "cutoff_len": 4096, + "generation_context_limit": 4096 +} diff --git a/perun/ministral_3b_aya/configs/input_config.yaml b/perun/ministral_3b_aya/configs/input_config.yaml new file mode 100644 index 0000000..a3f849b --- /dev/null +++ b/perun/ministral_3b_aya/configs/input_config.yaml @@ -0,0 +1,53 @@ +model_name_or_path: ministral/Ministral-3b-instruct +model_revision: 2c95908929198d6e69af8638f0dbbd9bc6b93f9e +stage: sft +do_train: true +do_eval: true +finetuning_type: lora +use_unsloth: true +use_unsloth_gc: true +bf16: true +infer_dtype: bfloat16 +flash_attn: sdpa +dataset_dir: data/aya_slovak_fixed +dataset: aya_train +eval_dataset: aya_eval +cutoff_len: 4096 +packing: false +train_on_prompt: false +preprocessing_num_workers: 8 +dataloader_num_workers: 2 +overwrite_cache: false +lora_rank: 16 +lora_alpha: 32 +lora_dropout: 0.0 +lora_target: q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj +per_device_train_batch_size: 8 +per_device_eval_batch_size: 8 +gradient_accumulation_steps: 1 +learning_rate: 0.0002 +num_train_epochs: 1.0 +lr_scheduler_type: cosine +warmup_ratio: 0.08 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 0.5 +gradient_checkpointing: true +ddp_find_unused_parameters: false +output_dir: /mnt/project/perun26012578/perun/models/ministral-3b-instruct/runs/full_aya_lr_0p000200_epoch1/adapter +overwrite_output_dir: false +logging_steps: 10 +eval_strategy: steps +eval_steps: 1000 +save_strategy: steps +save_steps: 1000 +save_total_limit: 2 +save_only_model: false +load_best_model_at_end: true +metric_for_best_model: loss +greater_is_better: false +predict_with_generate: false +plot_loss: true +report_to: none +seed: 42 +data_seed: 42 diff --git a/perun/ministral_3b_aya/configs/used_config.yaml b/perun/ministral_3b_aya/configs/used_config.yaml new file mode 100644 index 0000000..c353be0 --- /dev/null +++ b/perun/ministral_3b_aya/configs/used_config.yaml @@ -0,0 +1,53 @@ +model_name_or_path: ministral/Ministral-3b-instruct +model_revision: 2c95908929198d6e69af8638f0dbbd9bc6b93f9e +stage: sft +do_train: true +do_eval: true +finetuning_type: lora +use_unsloth: true +use_unsloth_gc: true +bf16: true +infer_dtype: bfloat16 +flash_attn: sdpa +dataset_dir: /mnt/project/perun26012578/perun/data/aya_slovak_fixed +dataset: aya_train +eval_dataset: aya_eval +cutoff_len: 4096 +packing: false +train_on_prompt: false +preprocessing_num_workers: 8 +dataloader_num_workers: 2 +overwrite_cache: false +lora_rank: 16 +lora_alpha: 32 +lora_dropout: 0.0 +lora_target: q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj +per_device_train_batch_size: 8 +per_device_eval_batch_size: 8 +gradient_accumulation_steps: 1 +learning_rate: 0.0002 +num_train_epochs: 1.0 +lr_scheduler_type: cosine +warmup_ratio: 0.08 +optim: adamw_torch +weight_decay: 0.0 +max_grad_norm: 0.5 +gradient_checkpointing: true +ddp_find_unused_parameters: false +output_dir: /mnt/project/perun26012578/perun/models/ministral-3b-instruct/runs/full_aya_lr_0p000200_epoch1/adapter +overwrite_output_dir: false +logging_steps: 10 +eval_strategy: steps +eval_steps: 1000 +save_strategy: steps +save_steps: 1000 +save_total_limit: 2 +save_only_model: false +load_best_model_at_end: true +metric_for_best_model: loss +greater_is_better: false +predict_with_generate: false +plot_loss: true +report_to: none +seed: 42 +data_seed: 42 diff --git a/perun/ministral_3b_aya/requirements/training.txt b/perun/ministral_3b_aya/requirements/training.txt new file mode 100644 index 0000000..eb6908b --- /dev/null +++ b/perun/ministral_3b_aya/requirements/training.txt @@ -0,0 +1,148 @@ +# Python 3.11.16; CUDA 12.4. +accelerate==1.11.0 +aiofiles==24.1.0 +aiohappyeyeballs==2.7.1 +aiohttp==3.14.3 +aiosignal==1.4.0 +alembic==1.20.0 +annotated-doc==0.0.5 +annotated-types==0.8.0 +antlr4-python3-runtime==4.9.3 +anyio==4.14.2 +attrs==26.1.0 +av==16.0.0 +bitsandbytes==0.50.2 +brotli==1.2.0 +certifi==2026.7.22 +charset-normalizer==3.5.1 +click==8.5.0 +colorlog==6.12.0 +contourpy==1.3.3 +cut-cross-entropy==25.1.1 +cycler==0.12.1 +datasets==3.6.0 +diffusers==0.40.0 +dill==0.3.8 +docstring_parser==0.18.0 +einops==0.8.2 +fastapi==0.141.1 +ffmpy==1.0.0 +filelock==3.32.3 +fire==0.7.1 +fonttools==4.63.0 +frozenlist==1.8.0 +fsspec==2025.3.0 +gradio==5.50.0 +gradio_client==1.14.0 +groovy==0.1.2 +h11==0.16.0 +hf-xet==1.6.0 +hf_transfer==0.1.9 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.29.0 +idna==3.19 +importlib_metadata==9.0.1 +Jinja2==3.1.6 +kiwisolver==1.5.1 +llamafactory @ git+https://github.com/hiyouga/LLaMA-Factory.git@7fcf5b3b130e5713b52415bb7404c476fada9c8c +Mako==1.4.3 +markdown-it-py==4.2.0 +MarkupSafe==3.0.3 +matplotlib==3.11.1 +mdurl==0.1.2 +modelscope==1.39.1 +modelscope-hub==0.3.0 +mpmath==1.3.0 +msgspec==0.21.1 +multidict==6.7.1 +multiprocess==0.70.16 +nest-asyncio==1.6.0 +networkx==3.6.1 +numpy==2.4.6 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-cusparselt-cu12==0.6.2 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +omegaconf==2.3.1 +optuna==5.0.0 +orjson==3.12.0 +packaging==26.3 +pandas==2.3.3 +peft==0.18.1 +pillow==11.3.0 +pip==26.2.1 +propcache==0.5.2 +protobuf==7.36.0 +psutil==7.2.2 +pyarrow==25.0.1 +pydantic==2.12.3 +pydantic_core==2.41.4 +pydub==0.25.1 +Pygments==2.21.0 +pyparsing==3.3.2 +python-dateutil==2.9.0.post0 +python-multipart==0.0.32 +pytz==2026.3.post1 +PyYAML==6.0.3 +regex==2026.7.19 +requests==2.34.2 +rich==15.0.0 +ruff==0.16.5 +safehttpx==0.1.7 +safetensors==0.8.0 +scipy==1.17.1 +semantic-version==2.10.0 +sentencepiece==0.2.2 +setuptools==80.10.2 +shellingham==1.5.4 +shtab==1.12.0 +six==1.17.0 +SQLAlchemy==2.1.4 +sse-starlette==3.4.8 +starlette==0.52.1 +sympy==1.13.1 +termcolor==3.3.0 +tiktoken==0.14.0 +tokenizers==0.22.2 +tomlkit==0.13.3 +torch==2.6.0+cu124 +torchaudio==2.6.0+cu124 +torchdata==0.11.0 +torchvision==0.21.0+cu124 +tqdm==4.70.0 +transformers==5.5.0 +triton==3.2.0 +trl==0.24.0 +typer==0.27.2 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +tyro==0.8.14 +tzdata==2026.3 +unsloth==2026.6.1 +unsloth_zoo==2026.8.16 +urllib3==2.7.0 +uvicorn==0.52.4 +websockets==15.0.1 +wheel==0.47.0 +xformers==0.0.29.post3 +xxhash==4.0.1 +yarl==1.24.5 +zipp==4.1.0 + +# Doplnenia pre generativnu evaluaciu a deklarovane zavislosti Zoo: +jieba==0.42.1 +rouge-chinese==1.0.3 +nltk==3.10.3 +joblib==1.6.0 +defusedxml==0.7.1 +torchao==0.13.0 diff --git a/perun/ministral_3b_aya/scripts/evaluate.py b/perun/ministral_3b_aya/scripts/evaluate.py new file mode 100644 index 0000000..07a9d7f --- /dev/null +++ b/perun/ministral_3b_aya/scripts/evaluate.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 +"""Evaluate the trained adapter on the prepared eval or test split; report loss, F1 and exact match.""" +from __future__ import annotations +import argparse +from collections import Counter, defaultdict +import hashlib +import json +import math +import os +from pathlib import Path +import subprocess +import sys +import unicodedata +from train import ROOT, read_config, write_config, verify_data, run_logged + +def normalized_tokens(text: str) -> list[str]: + text = unicodedata.normalize("NFKC", text).casefold() + return "".join( + " " if unicodedata.category(char).startswith("P") else char + for char in text + ).split() + + +def score_pair(prediction: str, reference: str) -> dict: + predicted = normalized_tokens(prediction) + expected = normalized_tokens(reference) + if not expected: + return { + "token_f1": None, + "normalized_exact_match": None, + "scored": False, + "skip_reason": "reference_has_zero_normalized_tokens", + "prediction_tokens": len(predicted), + "reference_tokens": 0, + } + shared = sum((Counter(predicted) & Counter(expected)).values()) + f1 = 200.0 * shared / (len(predicted) + len(expected)) + return { + "token_f1": f1, + "normalized_exact_match": 100.0 if predicted == expected else 0.0, + "scored": True, + "skip_reason": None, + "prediction_tokens": len(predicted), + "reference_tokens": len(expected), + } + + +def read_jsonl(path: Path) -> list[dict]: + result = [] + with path.open("r", encoding="utf-8-sig") as handle: + for line_number, line in enumerate(handle, 1): + if not line.strip(): + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError(f"{path}:{line_number}: invalid JSON: {exc.msg}") from exc + if not isinstance(row, dict): + raise ValueError(f"{path}:{line_number}: expected a JSON object") + result.append(row) + return result + + +def required_text(row: dict, field: str, context: str) -> str: + value = row.get(field) + if not isinstance(value, str): + raise ValueError(f"{context}: {field!r} must contain a string") + return value + + +def reference_text(row: dict, index: int) -> str: + for field in ("output", "response", "label"): + if field in row: + return required_text(row, field, f"reference row {index}") + raise ValueError(f"reference row {index}: missing output/response/label") + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def verify_training_dataset(adapter: Path, data_dir: Path) -> Path: + run_dir = adapter.parent + if adapter.name.startswith("checkpoint-") and not (run_dir / "run.json").is_file(): + run_dir = run_dir.parent + record_path = run_dir / "run.json" + if not record_path.is_file(): + raise ValueError(f"Missing training record: {record_path}. Keep the adapter with its original run files.") + record = json.loads(record_path.read_text(encoding="utf-8")) + if record.get("data_manifest_sha256") != sha256(data_dir / "manifest.json"): + raise ValueError("The prepared dataset differs from this adapter's training data. " + "Use the original dataset and its eval/test split recorded in run.json.") + return run_dir + + +def summarize(rows: list[dict]) -> dict: + scored = [row for row in rows if row["scored"]] + count = len(scored) + return { + "count_total": len(rows), + "count_scored": count, + "count_skipped_zero_token_references": len(rows) - count, + "count_empty_predictions_raw": sum(not row["prediction"].strip() for row in rows), + "count_zero_token_predictions": sum(row["prediction_tokens"] == 0 for row in rows), + "count_nonempty_predictions_raw": sum(bool(row["prediction"].strip()) for row in rows), + "count_truncated_or_modified_labels": sum( + not row["label_matches_full_reference"] for row in rows + ), + "count_labels_different_before_normalization": sum( + not row["label_matches_full_reference_raw"] for row in rows + ), + "token_f1": sum(row["token_f1"] for row in scored) / count if count else None, + "normalized_exact_match": ( + sum(row["normalized_exact_match"] for row in scored) / count if count else None + ), + "prediction_tokens_total": sum(row["prediction_tokens"] for row in rows), + "reference_tokens_total": sum(row["reference_tokens"] for row in rows), + } + + +def score_files( + predictions_path: Path, + references_path: Path, + output_dir: Path, + metadata_path: Path | None = None, +) -> dict: + predictions = read_jsonl(predictions_path) + references = read_jsonl(references_path) + if not references: + raise ValueError("The reference file contains no examples") + if len(predictions) != len(references): + raise ValueError( + f"Row count mismatch: {len(predictions)} predictions != {len(references)} references. " + "No scores written; evaluate the complete split in the original order." + ) + metadata = read_jsonl(metadata_path) if metadata_path is not None else [{} for _ in references] + if len(metadata) != len(references): + raise ValueError( + f"Row count mismatch: {len(metadata)} metadata rows != {len(references)} references" + ) + + rows = [] + groups = defaultdict(list) + for index, (prediction_row, reference_row, meta) in enumerate( + zip(predictions, references, metadata) + ): + prediction = required_text(prediction_row, "predict", f"prediction row {index}") + label = required_text(prediction_row, "label", f"prediction row {index}") + reference = reference_text(reference_row, index) + if "row_index" in meta and meta["row_index"] != index: + raise ValueError(f"metadata row {index}: row_index does not match the reference order") + source_metadata = meta.get("source_metadata", {}) + if not isinstance(source_metadata, dict): + raise ValueError(f"metadata row {index}: source_metadata must be an object") + task_type = meta.get( + "task_type", source_metadata.get("task_type", reference_row.get("task_type", "unknown")) + ) + if not isinstance(task_type, str): + raise ValueError(f"metadata row {index}: task_type must be a string") + row = { + "id": meta.get("stable_id", meta.get("id", reference_row.get("id", index))), + "index": index, + "task_type": task_type, + "prediction": prediction, + "reference": reference, + "exported_label": label, + "label_matches_full_reference": normalized_tokens(label) == normalized_tokens(reference), + "label_matches_full_reference_raw": label == reference, + **score_pair(prediction, reference), + } + rows.append(row) + groups[task_type].append(row) + + metrics = { + "schema_version": 1, + "metric_scale": "0..100", + "metric_version": "aya_overlap_v1", + "overall": summarize(rows), + "by_task_type": {name: summarize(group) for name, group in sorted(groups.items())}, + "inputs": { + "predictions": {"path": str(predictions_path.resolve()), "sha256": sha256(predictions_path)}, + "references": {"path": str(references_path.resolve()), "sha256": sha256(references_path)}, + }, + } + if metadata_path is not None: + metrics["inputs"]["metadata"] = {"path": str(metadata_path.resolve()), "sha256": sha256(metadata_path)} + + if output_dir.exists() and (not output_dir.is_dir() or any(output_dir.iterdir())): + raise ValueError(f"Output directory must be new or empty: {output_dir}") + output_dir.mkdir(parents=True, exist_ok=True) + with (output_dir / "per_sample.jsonl").open("w", encoding="utf-8", newline="\n") as handle: + for row in rows: + handle.write(json.dumps(row, ensure_ascii=False, allow_nan=False) + "\n") + (output_dir / "metrics.json").write_text( + json.dumps(metrics, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8" + ) + return metrics + + + +def evaluation_configs(training: dict, adapter: Path, output_dir: Path, split: str, max_new_tokens: int): + keys = ("model_name_or_path", "model_revision", "stage", "finetuning_type", "template", + "dataset_dir", "cutoff_len", "packing", "train_on_prompt", + "bf16", "infer_dtype", "flash_attn", "use_unsloth", "rope_scaling", "seed", "data_seed", + "preprocessing_num_workers", "dataloader_num_workers", "report_to") + base = {key: training[key] for key in keys if key in training} + base.update({"adapter_name_or_path": str(adapter), "eval_dataset": f"aya_{split}", + "use_unsloth_gc": False, "overwrite_output_dir": False}) + loss = dict(base, output_dir=str(output_dir / "loss"), do_eval=True, + predict_with_generate=False, per_device_eval_batch_size=8) + generation = dict(base, output_dir=str(output_dir / "generation"), do_predict=True, + predict_with_generate=True, per_device_eval_batch_size=1, + do_sample=False, num_beams=1, max_new_tokens=max_new_tokens) + return loss, generation + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("config", type=Path, help="The same model YAML used for training.") + parser.add_argument("--adapter", type=Path, help="Defaults to output_dir in the model YAML.") + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--split", choices=("eval", "test"), default="eval") + parser.add_argument("--max-new-tokens", type=int, default=256) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + if args.max_new_tokens < 1: + parser.error("--max-new-tokens must be positive.") + config = read_config(args.config.resolve()) + data_dir = Path(config["dataset_dir"]) + verify_data(data_dir, (args.split,)) + adapter = args.adapter.resolve() if args.adapter else Path(config["output_dir"]) + if not (adapter / "adapter_config.json").is_file(): + raise ValueError(f"Missing trained adapter: {adapter}") + run_dir = verify_training_dataset(adapter, data_dir) + used_config = run_dir / "used_config.yaml" + if not used_config.is_file(): + raise ValueError(f"Missing recorded training configuration: {used_config}") + if used_config.is_file(): + trained = read_config(used_config) + for key in ("model_name_or_path", "model_revision", "template", "cutoff_len", "rope_scaling"): + if trained.get(key) != config.get(key): + raise ValueError(f"Evaluation setting {key} differs from this adapter's training; use {used_config}.") + output_dir = args.output_dir.resolve() + if output_dir.exists() and any(output_dir.iterdir()): + raise ValueError(f"Evaluation output directory must be new or empty: {output_dir}") + loss, generation = evaluation_configs(config, adapter, output_dir, args.split, args.max_new_tokens) + if args.dry_run: + print(json.dumps({"split": args.split, "adapter": str(adapter), "evaluation_gpus": 1, + "loss_config": loss, "generation_config": generation}, ensure_ascii=False, indent=2)) + print("Evaluation configuration/data checks passed. No inference started.") + return 0 + output_dir.mkdir(parents=True, exist_ok=True) + record_path = output_dir / "evaluation.json" + record = {"status": "running", "split": args.split, "adapter": str(adapter), + "config": str(args.config.resolve()), "gpus": 1, "backend": "Unsloth", "precision": "bf16"} + record_path.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8") + env = os.environ.copy() + env.update({"PYTHONUNBUFFERED": "1", "TOKENIZERS_PARALLELISM": "false"}) + try: + for name, settings in (("eval_loss", loss), ("eval_generate", generation)): + path = output_dir / f"{name}.yaml" + write_config(path, settings) + command = [sys.executable, "-m", "torch.distributed.run", "--standalone", "--nproc_per_node=1", + str(ROOT / "scripts/train.py"), str(path), "--worker"] + run_logged(command, output_dir / f"{name}.log", env) + metrics = score_files(output_dir / "generation/generated_predictions.jsonl", + data_dir / f"{args.split}.jsonl", output_dir / "scores", + data_dir / f"{args.split}.metadata.jsonl") + reported = json.loads((output_dir / "loss/eval_results.json").read_text(encoding="utf-8")) + value = reported.get("eval_loss") + if value is None or not math.isfinite(value): + raise ValueError("LLaMA-Factory did not report a finite eval_loss.") + summary = {"model": config["model_name_or_path"], "split": args.split, + "eval_loss": value, "perplexity": math.exp(value) if value < 700 else None, + "token_f1": metrics["overall"]["token_f1"], + "normalized_exact_match": metrics["overall"]["normalized_exact_match"], + "examples": metrics["overall"]["count_total"], "metric_scale": "0..100", + "max_new_tokens": args.max_new_tokens, "cutoff_len": config["cutoff_len"], + "generation_context_limit": config["cutoff_len"]} + + (output_dir / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + record["status"] = "complete" + print(json.dumps(summary, ensure_ascii=False, indent=2)) + except BaseException: + record["status"] = "failed_or_interrupted" + raise + finally: + record_path.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (OSError, ValueError, RuntimeError, ImportError, subprocess.CalledProcessError) as error: + print(f"ERROR: {error}", file=sys.stderr) + raise SystemExit(1) diff --git a/perun/ministral_3b_aya/scripts/prepare_data.py b/perun/ministral_3b_aya/scripts/prepare_data.py new file mode 100644 index 0000000..0ed3436 --- /dev/null +++ b/perun/ministral_3b_aya/scripts/prepare_data.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 +"""Prepare one shared Slovak Aya train/eval/test dataset for LLaMA-Factory.""" + +from __future__ import annotations + +import argparse +from collections import Counter +from datetime import datetime, timezone +import hashlib +import json +from pathlib import Path +import sqlite3 +import sys +import unicodedata +from typing import Any, Iterator + + +HF_DATASET = "CohereLabs/aya_collection_language_split" +HF_REVISION = "a3af2fde4b4cb5b2775830b11244a1a20b5f004f" +HF_CONFIG = "slovak" +HF_SPLIT = "train" +SPLITS = ("train", "eval", "test") + + +def normalized(text: str, *, fold_case: bool = False) -> str: + text = " ".join(unicodedata.normalize("NFKC", text).split()) + return text.casefold() if fold_case else text + + +def digest_json(value: Any) -> str: + payload = json.dumps(value, ensure_ascii=False, separators=(",", ":")) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def file_sha256(path: Path) -> str: + result = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + result.update(chunk) + return result.hexdigest() + + +def write_json(path: Path, content: Any) -> None: + with path.open("x", encoding="utf-8", newline="\n") as target: + json.dump(content, target, ensure_ascii=False, indent=2, sort_keys=True) + target.write("\n") + + +def local_rows(path: Path) -> Iterator[Any]: + with path.open("r", encoding="utf-8-sig") as source: + for line_number, line in enumerate(source, 1): + if not line.strip(): + continue + try: + yield json.loads(line) + except json.JSONDecodeError as error: + raise ValueError(f"Invalid JSON at {path}:{line_number}: {error}") from error + + +def parquet_rows(parquet: Any) -> Iterator[dict[str, Any]]: + with parquet: + for batch in parquet.iter_batches(batch_size=2048): + yield from batch.to_pylist() + + +def open_source(args: argparse.Namespace) -> tuple[Iterator[Any], dict[str, Any]]: + local_path = args.local_parquet or args.local_jsonl + if local_path: + path = local_path.expanduser().resolve(strict=True) + if not path.is_file(): + raise ValueError(f"Local source is not a file: {path}") + source = { + "kind": "local_parquet" if args.local_parquet else "local_jsonl", + "path": str(path), + "sha256": file_sha256(path), + "bytes": path.stat().st_size, + "revision": None, + } + if args.local_parquet: + try: + from pyarrow.parquet import ParquetFile + except ImportError as error: + raise RuntimeError("Install pyarrow to read --local-parquet input.") from error + return parquet_rows(ParquetFile(path)), source + return local_rows(path), source + + try: + from datasets import load_dataset + from huggingface_hub import HfApi + except ImportError as error: + raise RuntimeError("Install datasets and huggingface_hub for Hugging Face input.") from error + + info = HfApi().dataset_info(HF_DATASET, revision=args.revision or HF_REVISION) + revision = info.sha + if not revision: + raise RuntimeError("Hugging Face did not return an immutable dataset revision.") + data = load_dataset( + HF_DATASET, HF_CONFIG, split=HF_SPLIT, revision=revision, streaming=True + ) + return iter(data), { + "kind": "huggingface", + "dataset": HF_DATASET, + "config": HF_CONFIG, + "split": HF_SPLIT, + "requested_revision": args.revision, + "revision": revision, + "streaming": True, + } + + +def convert_row(row: Any) -> tuple[dict[str, str], dict[str, Any]] | None: + if not isinstance(row, dict): + return None + if "inputs" in row or "targets" in row: + instruction, query, response = row.get("inputs"), "", row.get("targets") + excluded = {"inputs", "targets"} + else: + instruction, query, response = row.get("instruction"), row.get("input", ""), row.get("output") + query = "" if query is None else query + excluded = {"instruction", "input", "output"} + if not all(isinstance(value, str) for value in (instruction, query, response)): + return None + if not instruction.strip() or not response.strip(): + return None + metadata = {key: value for key, value in row.items() if key not in excluded} + return { + "instruction": instruction.strip(), + "input": query.strip(), + "output": response.strip(), + }, metadata + + +def create_database(path: Path) -> sqlite3.Connection: + connection = sqlite3.connect(path) + connection.executescript( + """ + PRAGMA journal_mode = DELETE; + PRAGMA synchronous = NORMAL; + PRAGMA temp_store = FILE; + PRAGMA cache_size = -32768; + CREATE TABLE prompt_groups ( + group_id TEXT PRIMARY KEY, + order_key TEXT NOT NULL, + size INTEGER NOT NULL, + split TEXT NOT NULL DEFAULT 'train' + ); + CREATE TABLE records ( + position INTEGER PRIMARY KEY, + stable_id TEXT NOT NULL UNIQUE, + group_id TEXT NOT NULL, + instruction TEXT NOT NULL, + input TEXT NOT NULL, + output TEXT NOT NULL, + metadata TEXT NOT NULL, + source_row_index INTEGER NOT NULL, + source_dataset TEXT NOT NULL, + source_task TEXT NOT NULL + ); + """ + ) + return connection + + +def ingest(connection: sqlite3.Connection, rows: Iterator[Any], seed: int) -> Counter: + counts: Counter = Counter() + for source_index, row in enumerate(rows): + counts["source_rows"] += 1 + converted = convert_row(row) + if converted is None: + counts["invalid_or_empty_rows"] += 1 + else: + data, metadata = converted + prompt_key = [normalized(data["instruction"], fold_case=True), normalized(data["input"], fold_case=True)] + group_id = digest_json(prompt_key) + stable_id = digest_json([prompt_key, normalized(data["output"])]) + cursor = connection.execute( + """INSERT OR IGNORE INTO records + (stable_id, group_id, instruction, input, output, metadata, + source_row_index, source_dataset, source_task) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""", + (stable_id, group_id, data["instruction"], data["input"], data["output"], + json.dumps(metadata, ensure_ascii=False, separators=(",", ":")), source_index, + str(metadata.get("dataset_name") or "unknown"), str(metadata.get("task_type") or "unknown"))) + if cursor.rowcount == 0: + counts["duplicate_pairs_removed"] += 1 + else: + counts["retained_rows"] += 1 + connection.execute( + """INSERT INTO prompt_groups (group_id, order_key, size) VALUES (?, ?, 1) + ON CONFLICT(group_id) DO UPDATE SET size = size + 1""", + (group_id, digest_json([seed, group_id]))) + if counts["source_rows"] % 10000 == 0: + connection.commit() + print(f"Read {counts['source_rows']:,}; retained {counts['retained_rows']:,}", flush=True) + connection.commit() + for key in ("source_rows", "invalid_or_empty_rows", "duplicate_pairs_removed", "retained_rows"): + counts.setdefault(key, 0) + if not counts["retained_rows"]: + raise ValueError("The source contains no usable examples after validation.") + connection.execute("CREATE INDEX prompt_group_order ON prompt_groups(order_key, group_id)") + connection.execute("CREATE INDEX records_group ON records(group_id)") + connection.commit() + counts["prompt_groups"] = connection.execute("SELECT COUNT(*) FROM prompt_groups").fetchone()[0] + counts["groups_with_multiple_responses"] = connection.execute("SELECT COUNT(*) FROM prompt_groups WHERE size > 1").fetchone()[0] + return counts + + +def assign_splits(connection: sqlite3.Connection, eval_size: int, test_size: int) -> dict[str, dict[str, int]]: + remaining = {"eval": eval_size, "test": test_size} + assignments: list[tuple[str, str]] = [] + # Selection depends only on seed, normalized prompt, and the number of its + # distinct responses. An entire prompt group always stays in one split. + for group_id, size in connection.execute( + "SELECT group_id, size FROM prompt_groups ORDER BY order_key, group_id" + ): + split = next((name for name in ("eval", "test") if size <= remaining[name]), None) + if split is not None: + remaining[split] -= size + assignments.append((split, group_id)) + if not any(remaining.values()): + break + connection.executemany("UPDATE prompt_groups SET split = ? WHERE group_id = ?", assignments) + connection.commit() + result = {name: {"rows": 0, "prompt_groups": 0} for name in SPLITS} + for split, group_count, row_count in connection.execute( + "SELECT split, COUNT(*), SUM(size) FROM prompt_groups GROUP BY split" + ): + result[split] = {"rows": row_count, "prompt_groups": group_count} + if not result["train"]["rows"]: + raise ValueError("No rows remain for train. Reduce --eval-size and/or --test-size.") + # A record belongs to one prompt group; each group has exactly one split. + conflicts = connection.execute( + """SELECT COUNT(*) FROM ( + SELECT r.group_id FROM records r JOIN prompt_groups g USING(group_id) + GROUP BY r.group_id HAVING COUNT(DISTINCT g.split) != 1 + )""" + ).fetchone()[0] + if conflicts: + raise RuntimeError("Prompt-disjoint split verification failed.") + if result["eval"]["rows"] > eval_size or result["test"]["rows"] > test_size: + raise RuntimeError("A holdout exceeds its requested row limit.") + return result + + +def emit_splits(connection: sqlite3.Connection, output: Path, expected: dict[str, dict[str, int]]) -> dict[str, Any]: + files: dict[str, Any] = {} + for split in SPLITS: + data_path = output / f"{split}.jsonl" + metadata_path = output / f"{split}.metadata.jsonl" + row_count = 0 + with data_path.open("x", encoding="utf-8", newline="\n") as data_file, metadata_path.open( + "x", encoding="utf-8", newline="\n" + ) as metadata_file: + cursor = connection.execute( + """SELECT r.stable_id, r.group_id, r.instruction, r.input, r.output, + r.metadata, r.source_row_index + FROM records r JOIN prompt_groups g USING(group_id) + WHERE g.split = ? ORDER BY g.order_key, r.stable_id""", (split,) + ) + for stable_id, group_id, instruction, query, response, metadata, source_index in cursor: + item = {"instruction": instruction, "input": query, "output": response} + provenance = { + "row_index": row_count, + "stable_id": stable_id, + "prompt_group_id": group_id, + "source_row_index": source_index, + "source_metadata": json.loads(metadata), + } + provenance["source_id"] = provenance["source_metadata"].get("id") + data_file.write(json.dumps(item, ensure_ascii=False, separators=(",", ":")) + "\n") + metadata_file.write(json.dumps(provenance, ensure_ascii=False, separators=(",", ":")) + "\n") + row_count += 1 + if row_count != expected[split]["rows"]: + raise RuntimeError(f"Unexpected {split} output count: {row_count}") + for path in (data_path, metadata_path): + files[path.name] = {"sha256": file_sha256(path), "bytes": path.stat().st_size, "rows": row_count} + return files + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--revision", help="Hugging Face revision; resolved to an immutable commit before loading.") + local_input = parser.add_mutually_exclusive_group() + local_input.add_argument("--local-jsonl", type=Path, help="Offline JSONL input in Aya or Alpaca format.") + local_input.add_argument("--local-parquet", type=Path, help="Offline Parquet input in Aya or Alpaca format.") + parser.add_argument("--eval-size", type=int, default=1000, help="Maximum number of eval records (default: 1000).") + parser.add_argument("--test-size", type=int, default=1000, help="Maximum number of test records (default: 1000).") + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--keep-db", action="store_true", help="Keep the temporary SQLite database.") + args = parser.parse_args() + if min(args.eval_size, args.test_size) < 0: + parser.error("Holdout sizes must be nonnegative.") + if (args.local_jsonl or args.local_parquet) and args.revision: + parser.error("--revision applies only to Hugging Face input.") + output = args.output_dir.expanduser().resolve() + if output.exists() and (not output.is_dir() or any(output.iterdir())): + parser.error(f"Output must be a new or empty directory: {output}") + # Resolve source before creating output, so authentication or dependency + # errors do not produce a misleading empty data directory. + rows, source = open_source(args) + output.mkdir(parents=True, exist_ok=True) + database_path = output / "preparation.sqlite3" + connection = create_database(database_path) + try: + counts = ingest(connection, rows, args.seed) + splits = assign_splits(connection, args.eval_size, args.test_size) + if sum(item["rows"] for item in splits.values()) != counts["retained_rows"]: + raise RuntimeError("Split counts do not sum to the retained record count.") + files = emit_splits(connection, output, splits) + registration = { + f"aya_{split}": { + "file_name": f"{split}.jsonl", "formatting": "alpaca", + "columns": {"prompt": "instruction", "query": "input", "response": "output"}, + } for split in SPLITS + } + info_path = output / "dataset_info.json" + write_json(info_path, registration) + files[info_path.name] = {"sha256": file_sha256(info_path), "bytes": info_path.stat().st_size} + breakdown = [ + {"split": split, "dataset_name": dataset, "task_type": task, "rows": rows_count, "prompt_groups": groups_count} + for split, dataset, task, rows_count, groups_count in connection.execute( + """SELECT g.split, r.source_dataset, r.source_task, COUNT(*), COUNT(DISTINCT r.group_id) + FROM records r JOIN prompt_groups g USING(group_id) + GROUP BY g.split, r.source_dataset, r.source_task + ORDER BY g.split, r.source_dataset, r.source_task""" + ) + ] + manifest = { + "format_version": 1, + "prepared_at_utc": datetime.now(timezone.utc).isoformat(), + "preparation_script_sha256": file_sha256(Path(__file__).resolve()), + "source": source, + "seed": args.seed, + "counts": dict(counts), + "requested_holdout_max_rows": {"eval": args.eval_size, "test": args.test_size}, + "splits": splits, + "source_breakdown": breakdown, + "files": files, + } + write_json(output / "manifest.json", manifest) + finally: + connection.close() + if not args.keep_db: + # Only our explicitly named staging file inside this new output is removed. + database_path.unlink() + print(json.dumps({"output_dir": str(output), "source": source, "counts": dict(counts), "splits": splits}, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (ValueError, RuntimeError) as error: + print(f"ERROR: {error}", file=sys.stderr) + raise SystemExit(1) diff --git a/perun/ministral_3b_aya/scripts/train.py b/perun/ministral_3b_aya/scripts/train.py new file mode 100644 index 0000000..37064b2 --- /dev/null +++ b/perun/ministral_3b_aya/scripts/train.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +"""Train from one LLaMA-Factory YAML on four GPUs; import Unsloth in each worker first.""" +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import functools +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +from types import MethodType + +ROOT = Path(__file__).resolve().parents[1] +GPUS = 4 + + +def read_config(path: Path) -> dict: + import yaml + with path.open(encoding="utf-8-sig") as handle: + config = yaml.safe_load(handle) + if not isinstance(config, dict): + raise ValueError("The configuration must be a YAML mapping.") + if config.get("use_unsloth") is not True: + raise ValueError("This package requires use_unsloth: true.") + if config.get("bf16") is not True or config.get("fp16", False): + raise ValueError("The H200 configuration must use bf16, not fp16.") + if "model_max_length" in config: + raise ValueError("Remove model_max_length from YAML; LLaMA-Factory derives it from cutoff_len.") + if int(config.get("cutoff_len", 0)) < 1: + raise ValueError("cutoff_len must be positive.") + for key in ("dataset_dir", "output_dir"): + if key not in config: + raise ValueError(f"Missing configuration key: {key}") + value = Path(config[key]).expanduser() + config[key] = str((ROOT / value).resolve() if not value.is_absolute() else value.resolve()) + return config + + +def write_config(path: Path, config: dict) -> None: + import yaml + path.write_text(yaml.safe_dump(config, allow_unicode=True, sort_keys=False), encoding="utf-8") + + +def digest(path: Path) -> str: + result = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + result.update(block) + return result.hexdigest() + + +def verify_data(data_dir: Path, splits: tuple[str, ...]) -> dict: + manifest_path = data_dir / "manifest.json" + if not manifest_path.is_file(): + raise ValueError(f"Run prepare_data.py first; missing {manifest_path}") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + for split in splits: + if manifest.get("splits", {}).get(split, {}).get("rows", 0) < 1: + raise ValueError(f"The prepared {split} split is empty.") + names = ["dataset_info.json"] + for split in splits: + names.extend((f"{split}.jsonl", f"{split}.metadata.jsonl")) + for name in names: + item = manifest.get("files", {}).get(name, {}) + if not item.get("sha256") or digest(data_dir / name) != item["sha256"]: + raise ValueError(f"Prepared data changed or is missing: {name}") + return manifest + + +def run_logged(command: list[str], log: Path, env: dict | None = None) -> None: + print("Running:", " ".join(command), flush=True) + with log.open("a", encoding="utf-8") as handle: + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, encoding="utf-8", errors="replace", env=env) + try: + for line in process.stdout: + print(line, end="", flush=True) + handle.write(line) + handle.flush() + code = process.wait() + except BaseException: + process.terminate() + process.wait() + raise + if code: + raise subprocess.CalledProcessError(code, command) + + +def context_settings(settings: dict, original_config) -> dict: + requested = int(settings["cutoff_len"]) + native = int(original_config.max_position_embeddings) + if requested > native or settings.get("rope_scaling"): + raise ValueError(f"Use cutoff_len <= {native} within the model's native context and omit rope_scaling.") + return {"requested_length": requested, "base_positions": native} + + +def reserve_generation_space() -> None: + # Reserve space before batching so the trainer sees the same prompt length + # as model.generate when it masks the prompt in the generated sequence. + from llamafactory.data.processor import unsupervised + + original = unsupervised.infer_seqlen + + @functools.wraps(original) + def with_generation_space(source_len: int, target_len: int, cutoff_len: int): + source, target = original(source_len, target_len, cutoff_len) + source = min(source, cutoff_len - 1) + if source < source_len or target < target_len: + print(f"Generation preprocessing truncated tokens: input {source_len} -> {source}, " + f"label {target_len} -> {target}, context {cutoff_len}.", flush=True) + return source, target + + unsupervised.infer_seqlen = with_generation_space + + +def bound_generation_context(model, context_length: int, default_max_new_tokens: int) -> None: + original_generate = model.generate + + @functools.wraps(original_generate) + def bounded_generate(self, *args, **kwargs): + inputs = kwargs.get("input_ids") + if inputs is None: + inputs = kwargs.get("inputs") + if inputs is None and args: + inputs = args[0] + if inputs is None: + raise ValueError("Cannot verify the text input length before generation.") + prompt_length = int(inputs.shape[-1]) + available = context_length - prompt_length + if available < 1: + raise ValueError("The preprocessed input leaves no room for an answer inside the context limit.") + requested = kwargs.get("max_new_tokens") + requested = int(default_max_new_tokens if requested is None else requested) + if requested < 1: + raise ValueError("max_new_tokens must be positive.") + kwargs["max_new_tokens"] = min(requested, available) + if requested > available: + print(f"Generation limit: {available} new tokens after {prompt_length} input tokens " + f"(context {context_length}).", flush=True) + return original_generate(*args, **kwargs) + + model.generate = MethodType(bounded_generate, model) + + +def require_unsloth_backend(settings: dict, context: dict) -> dict: + # LLaMA-Factory may otherwise silently fall back when Unsloth rejects a model. + # The wrappers affect this process only; no installed package is edited. + from llamafactory.model import loader + from llamafactory.model.model_utils import unsloth as integration + loaded = {} + + if hasattr(integration, "_get_unsloth_kwargs"): + original_kwargs = integration._get_unsloth_kwargs + + @functools.wraps(original_kwargs) + def pinned_kwargs(config, model_name_or_path, model_args, finetuning_args): + values = original_kwargs(config, model_name_or_path, model_args, finetuning_args) + values["use_exact_model_name"] = True + if model_args.model_revision: + values["revision"] = model_args.model_revision + return values + + integration._get_unsloth_kwargs = pinned_kwargs + + def strict(original): + @functools.wraps(original) + def wrapped(*args, **kwargs): + model = original(*args, **kwargs) + if model is None: + raise RuntimeError("Unsloth did not load this model. Training/evaluation stopped; " + "use a compatible Unsloth/LLaMA-Factory environment.") + if model.config.max_position_embeddings < context["requested_length"]: + raise RuntimeError("The loaded model's configured context is shorter than the requested SFT context.") + if settings.get("do_train"): + loaded["base_config_json"] = model.config.to_json_string() + return model + return wrapped + + for name in ("load_unsloth_pretrained_model", "load_unsloth_peft_model"): + original = getattr(loader, name, None) + if original is None: + raise RuntimeError("The installed LLaMA-Factory has an unsupported Unsloth integration.") + setattr(loader, name, strict(original)) + if settings.get("do_predict") and settings.get("predict_with_generate"): + # Keep PEFT/Unsloth delegation: GenerationMixin.generate bound directly + # to a PeftModel fails in Transformers 5 because its class has no _sample. + original_patch_model = loader.patch_model + + @functools.wraps(original_patch_model) + def patch_model_preserving_generate(model, *args, **kwargs): + original_generate = model.generate + result = original_patch_model(model, *args, **kwargs) + model.generate = original_generate + return result + + loader.patch_model = patch_model_preserving_generate + # Attach the guard to the final model, after LLaMA-Factory patches generate + # and finishes adapter initialization. Earlier attachment can be replaced. + from llamafactory.train.sft import workflow + original_load_model = workflow.load_model + + @functools.wraps(original_load_model) + def load_bounded_model(*args, **kwargs): + model = original_load_model(*args, **kwargs) + bound_generation_context(model, context["requested_length"], int(settings["max_new_tokens"])) + return model + + workflow.load_model = load_bounded_model + return loaded + + +def worker(config_path: Path) -> None: + # This order is important for Unsloth's patches. + import unsloth # noqa: F401 + import torch + config = read_config(config_path) + world = int(os.environ.get("WORLD_SIZE", "1")) + if config.get("do_train") and world != GPUS: + raise RuntimeError("This training scenario requires four processes / four GPUs.") + if not torch.cuda.is_available() or torch.cuda.device_count() < world: + raise RuntimeError(f"This job requires {world} visible CUDA GPU(s).") + local_rank = int(os.environ.get("LOCAL_RANK", "0")) + torch.cuda.set_device(local_rank) + if not torch.cuda.is_bf16_supported(): + raise RuntimeError("The allocated GPU does not support the requested bf16 precision.") + print(f"rank={local_rank}, world_size={world}, GPU={torch.cuda.get_device_name(local_rank)}, " + "precision=bf16, backend=Unsloth", flush=True) + from transformers import AutoConfig + original_config = AutoConfig.from_pretrained(config["model_name_or_path"], revision=config["model_revision"], trust_remote_code=False) + context = context_settings(config, original_config) + print("Model context:", json.dumps(context), flush=True) + loaded = require_unsloth_backend(config, context) + if config.get("do_predict") and config.get("predict_with_generate"): + if context["requested_length"] < 2: + raise ValueError("Generation requires cutoff_len >= 2.") + reserve_generation_space() + from llamafactory.train.tuner import run_exp + run_exp(config) + if config.get("do_train") and local_rank == 0: + (Path(config["output_dir"]) / "base_model_config.json").write_text(loaded["base_config_json"], encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("config", type=Path, help="LLaMA-Factory model YAML.") + parser.add_argument("--run-dir", type=Path, help="New run directory; adapter is saved inside adapter/.") + parser.add_argument("--resume", type=Path, help="Checkpoint inside the original run's adapter/ directory.") + parser.add_argument("--pilot-steps", type=int, help="Limit a pilot to this many optimizer steps; requires --run-dir.") + parser.add_argument("--pilot-samples", type=int, + help="Maximum rows per dataset in a pilot (default: 2048 or all eval rows, whichever is larger).") + parser.add_argument("--pilot-eval-steps", type=int, + help="Evaluate and save at this interval in a pilot (default: at the last step).") + parser.add_argument("--dry-run", action="store_true", help="Validate data/config; do not import GPU libraries.") + parser.add_argument("--worker", action="store_true", help=argparse.SUPPRESS) + args = parser.parse_args() + if args.worker: + worker(args.config.resolve()) + return 0 + if args.pilot_steps is None and args.pilot_samples is not None: + parser.error("--pilot-samples requires --pilot-steps.") + if args.pilot_steps is not None and (args.pilot_steps < 1 or args.run_dir is None): + parser.error("--pilot-steps must be positive and requires --run-dir.") + if args.pilot_samples is not None and args.pilot_samples < 1: + parser.error("--pilot-samples must be positive.") + if args.pilot_eval_steps is not None and not 1 <= args.pilot_eval_steps <= (args.pilot_steps or 0): + parser.error("--pilot-eval-steps requires --pilot-steps and must be between 1 and that limit.") + config = read_config(args.config.resolve()) + if config.get("stage") != "sft" or config.get("finetuning_type") != "lora": + raise ValueError("Expected an SFT / LoRA training configuration.") + if not config.get("do_train") or config.get("eval_dataset") != "aya_eval": + raise ValueError("Training must use the prepared train and eval splits.") + if config.get("dataset") != "aya_train" or "val_size" in config: + raise ValueError("Use aya_train and aya_eval without another automatic split.") + if any(key in config for key in ("max_steps", "max_samples")): + raise ValueError("The full-data scenario must not impose a smoke-test sample/step limit.") + batch = GPUS * config["per_device_train_batch_size"] * config["gradient_accumulation_steps"] + data_dir = Path(config["dataset_dir"]) + manifest = verify_data(data_dir, ("train", "eval")) + train_rows = manifest["splits"]["train"]["rows"] + eval_rows = manifest["splits"]["eval"]["rows"] + if args.pilot_steps is not None: + # LLaMA-Factory limits both splits before tokenization. Keep the complete + # validation split, even if the caller prepared more than 2048 eval rows. + sample_limit = args.pilot_samples or max(2048, eval_rows) + if sample_limit < eval_rows: + parser.error(f"--pilot-samples must be at least {eval_rows} to retain the full eval split.") + eval_interval = args.pilot_eval_steps or args.pilot_steps + config.update(max_steps=args.pilot_steps, max_samples=sample_limit, + eval_strategy="steps", save_strategy="steps", + eval_steps=eval_interval, save_steps=eval_interval) + if args.run_dir: + config["output_dir"] = str(args.run_dir.resolve() / "adapter") + run_dir = Path(config["output_dir"]).parent + if args.resume: + checkpoint = args.resume.resolve() + checkpoint.relative_to(Path(config["output_dir"])) + for name in ("trainer_state.json", "adapter_config.json", "optimizer.pt", "scheduler.pt"): + if not (checkpoint / name).is_file(): + raise ValueError(f"Incomplete checkpoint: {checkpoint / name}") + previous = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) + if previous["config"] != config or previous["data_manifest_sha256"] != digest(data_dir / "manifest.json"): + raise ValueError("Resume requires the original configuration and prepared data.") + elif run_dir.exists() and any(run_dir.iterdir()): + raise ValueError(f"Run directory is not empty; choose --run-dir for a new run: {run_dir}") + summary = {"gpus_per_training_run": GPUS, "effective_batch": batch, + "cutoff_len": config["cutoff_len"], "bf16": True, "use_unsloth": True, + "train_rows": train_rows, "eval_rows": eval_rows, + "adapter_output": config["output_dir"]} + if args.pilot_steps is not None: + summary.update(pilot_steps=args.pilot_steps, pilot_train_rows=min(train_rows, config["max_samples"]), + pilot_eval_steps=config["eval_steps"]) + print(json.dumps(summary, ensure_ascii=False, indent=2)) + if args.dry_run: + print("Configuration/data checks passed. No training started.") + return 0 + run_dir.mkdir(parents=True, exist_ok=True) + if not args.resume: + record = {"config": config, "gpus": GPUS, "effective_batch": batch, + "data_manifest_sha256": digest(data_dir / "manifest.json"), + "created_at_utc": datetime.now(timezone.utc).isoformat()} + (run_dir / "run.json").write_text(json.dumps(record, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + execution = dict(config) + if args.resume: + execution["resume_from_checkpoint"] = str(args.resume.resolve()) + execution_path = run_dir / "used_config.yaml" + write_config(execution_path, execution) + env = os.environ.copy() + env.update({"PYTHONUNBUFFERED": "1", "TOKENIZERS_PARALLELISM": "false"}) + env.setdefault("OMP_NUM_THREADS", "4") + command = [sys.executable, "-m", "torch.distributed.run", "--standalone", + f"--nproc_per_node={GPUS}", str(Path(__file__).resolve()), + str(execution_path), "--worker"] + run_logged(command, run_dir / "train.log", env) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (OSError, ValueError, RuntimeError, ImportError, subprocess.CalledProcessError) as error: + print(f"ERROR: {error}", file=sys.stderr) + raise SystemExit(1) diff --git a/perun/ministral_3b_aya/sk_bench/aggregated/models__ministral-base.json b/perun/ministral_3b_aya/sk_bench/aggregated/models__ministral-base.json new file mode 100644 index 0000000..f39cf98 --- /dev/null +++ b/perun/ministral_3b_aya/sk_bench/aggregated/models__ministral-base.json @@ -0,0 +1,3250 @@ +{ + "model_name": "models/ministral-base", + "config_general": { + "lighteval_sha": "?", + "num_fewshot_seeds": 1, + "max_samples": null, + "job_id": "0", + "start_time": 883355.287443326, + "end_time": 883520.97395618, + "total_evaluation_time_secondes": "165.68651285406668", + "model_config": { + "model_name": "models/ministral-base", + "generation_parameters": { + "num_blocks": null, + "block_size": null, + "early_stopping": null, + "repetition_penalty": null, + "frequency_penalty": null, + "length_penalty": null, + "presence_penalty": null, + "max_new_tokens": null, + "min_new_tokens": null, + "seed": null, + "stop_tokens": null, + "temperature": 0, + "top_k": null, + "min_p": null, + "top_p": null, + "truncate_prompt": null, + "cache_implementation": null, + "response_format": null, + "skip_special_tokens": true + }, + "system_prompt": null, + "cache_dir": "~/.cache/huggingface/lighteval", + "tokenizer": null, + "revision": "main", + "dtype": "bfloat16", + "tensor_parallel_size": 1, + "data_parallel_size": 1, + "pipeline_parallel_size": 1, + "gpu_memory_utilization": 0.9, + "enable_prefix_caching": null, + "max_model_length": null, + "quantization": null, + "load_format": null, + "swap_space": 4, + "seed": 1234, + "trust_remote_code": true, + "add_special_tokens": true, + "multichoice_continuations_start_space": true, + "pairwise_tokenization": false, + "max_num_seqs": 128, + "max_num_batched_tokens": 2048, + "subfolder": null, + "is_async": false, + "override_chat_template": null, + "enforce_eager": false + }, + "model_name": "models/ministral-base" + }, + "versions": {}, + "results": { + "e_tests.prompt1_mqa_single_correct_sk:slovensky-jazyk|0": { + "acc": 0.25109170305676853, + "acc_stderr": 0.020284878098207157, + "em_relaxed": 0.25327510917030566, + "em_relaxed_stderr": 0.020343162564319364 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0": { + "acc": 0.19638242894056848, + "acc_stderr": 0.020220056896337255, + "em_relaxed": 0.19896640826873385, + "em_relaxed_stderr": 0.02031990120797851 + }, + "mmlu.prompt2_mqa_single_correct_sk:philosophy|0": { + "acc": 0.12337662337662338, + "acc_stderr": 0.01876954635295657, + "em_relaxed": 0.12337662337662338, + "em_relaxed_stderr": 0.01876954635295657 + }, + "scio.prompt1_mqa_single_correct_sk:mat|0": { + "acc": 0.23497267759562843, + "acc_stderr": 0.022192218701649325, + "em_relaxed": 0.2459016393442623, + "em_relaxed_stderr": 0.02253970761756048 + }, + "mmlu.prompt3_mqa_single_correct_sk:security_studies|0": { + "acc": 0.1981981981981982, + "acc_stderr": 0.026815582053277193, + "em_relaxed": 0.20270270270270271, + "em_relaxed_stderr": 0.027042309486278813 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0": { + "acc": 0.20522388059701493, + "acc_stderr": 0.017460611985170203, + "em_relaxed": 0.21082089552238806, + "em_relaxed_stderr": 0.017634685741009386 + }, + "mmlu.prompt2_mqa_single_correct_sk:security_studies|0": { + "acc": 0.16666666666666666, + "acc_stderr": 0.02506903489984115, + "em_relaxed": 0.1891891891891892, + "em_relaxed_stderr": 0.02634582401273633 + }, + "mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0": { + "acc": 0.1891891891891892, + "acc_stderr": 0.02438361375395457, + "em_relaxed": 0.1891891891891892, + "em_relaxed_stderr": 0.02438361375395457 + }, + "mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0": { + "acc": 0.2644927536231884, + "acc_stderr": 0.026597083702337265, + "em_relaxed": 0.26811594202898553, + "em_relaxed_stderr": 0.02671259720934512 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0": { + "acc": 0.2878787878787879, + "acc_stderr": 0.03225883512300992, + "em_relaxed": 0.2878787878787879, + "em_relaxed_stderr": 0.03225883512300992 + }, + "mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0": { + "acc": 0.17, + "acc_stderr": 0.03775251680686371, + "em_relaxed": 0.17, + "em_relaxed_stderr": 0.03775251680686371 + }, + "slovak_tests.prompt3_qa_sk:maturita|0": { + "qem": 0.01327433628318584, + "qem_stderr": 0.007629803632081149, + "qf1": 0.0022123893805309734, + "qf1_stderr": 0.0015609152277667225, + "rouge1": 0.0013282561753440866, + "rouge1_stderr": 0.0009107295550233445, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0013282561753440866, + "rougeL_stderr": 0.0009107295550233445 + }, + "slovak_tests.prompt1_mqa_single_correct_sk:monitor9|0": { + "acc": 0.24423963133640553, + "acc_stderr": 0.029232990766796058, + "em_relaxed": 0.24423963133640553, + "em_relaxed_stderr": 0.029232990766796058 + }, + "mmlu.prompt2_mqa_single_correct_sk:college_medicine|0": { + "acc": 0.25, + "acc_stderr": 0.033113308926626096, + "em_relaxed": 0.25, + "em_relaxed_stderr": 0.033113308926626096 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0": { + "acc": 0.3939393939393939, + "acc_stderr": 0.04935824351078519, + "em_relaxed": 0.41414141414141414, + "em_relaxed_stderr": 0.04975740158605099 + }, + "mmlu.prompt3_mqa_single_correct_sk:machine_learning|0": { + "acc": 0.16964285714285715, + "acc_stderr": 0.035623678500953895, + "em_relaxed": 0.1875, + "em_relaxed_stderr": 0.0370468111477387 + }, + "mmlu.prompt2_mqa_single_correct_sk:virology|0": { + "acc": 0.17682926829268292, + "acc_stderr": 0.02988327785748596, + "em_relaxed": 0.18902439024390244, + "em_relaxed_stderr": 0.030666839281449514 + }, + "mmlu.prompt2_mqa_single_correct_sk:anatomy|0": { + "acc": 0.2462686567164179, + "acc_stderr": 0.037358294208002164, + "em_relaxed": 0.2537313432835821, + "em_relaxed_stderr": 0.037731914327133735 + }, + "mmlu.prompt1_mqa_single_correct_sk:global_facts|0": { + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "em_relaxed": 0.3, + "em_relaxed_stderr": 0.046056618647183814 + }, + "e_tests.prompt2_mqa_single_correct_sk:geografia|0": { + "acc": 0.2857142857142857, + "acc_stderr": 0.06520506636966263, + "em_relaxed": 0.2857142857142857, + "em_relaxed_stderr": 0.06520506636966263 + }, + "e_tests.prompt2_mqa_single_correct_sk:chemia|0": { + "acc": 0.07142857142857142, + "acc_stderr": 0.034726602486028414, + "em_relaxed": 0.08928571428571429, + "em_relaxed_stderr": 0.038450387280282494 + }, + "mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0": { + "acc": 0.1656441717791411, + "acc_stderr": 0.020621585415044594, + "em_relaxed": 0.17484662576687116, + "em_relaxed_stderr": 0.021069502803436863 + }, + "mmlu.prompt3_mqa_single_correct_sk:econometrics|0": { + "acc": 0.1875, + "acc_stderr": 0.0370468111477387, + "em_relaxed": 0.19642857142857142, + "em_relaxed_stderr": 0.03770970049347017 + }, + "movie_review.prompt2_sentiment_binary_sk|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "e_tests.prompt1_mqa_single_correct_sk:biologia|0": { + "acc": 0.19540229885057472, + "acc_stderr": 0.0427567811097387, + "em_relaxed": 0.20689655172413793, + "em_relaxed_stderr": 0.04368097459950702 + }, + "mmlu.prompt3_mqa_single_correct_sk:management|0": { + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "em_relaxed": 0.21, + "em_relaxed_stderr": 0.040936018074033256 + }, + "e_tests.prompt3_mqa_multiple_correct_sk:fyzika|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "mmlu.prompt2_mqa_single_correct_sk:formal_logic|0": { + "acc": 0.1565217391304348, + "acc_stderr": 0.034030776100725475, + "em_relaxed": 0.2, + "em_relaxed_stderr": 0.03746343246326772 + }, + "mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0": { + "acc": 0.22, + "acc_stderr": 0.04163331998932268, + "em_relaxed": 0.22, + "em_relaxed_stderr": 0.04163331998932268 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0": { + "acc": 0.22727272727272727, + "acc_stderr": 0.029857515673386393, + "em_relaxed": 0.23232323232323232, + "em_relaxed_stderr": 0.030088629490217483 + }, + "movie_review.prompt3_sentiment_binary_sk|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slovak_tests.prompt2_qa_sk:maturita|0": { + "qem": 0.01327433628318584, + "qem_stderr": 0.007629803632081149, + "qf1": 0.002753195673549656, + "qf1_stderr": 0.0020208822149325673, + "rouge1": 0.0008509189925119129, + "rouge1_stderr": 0.0007009516038640078, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0008509189925119129, + "rougeL_stderr": 0.0007009516038640078 + }, + "e_tests.prompt1_qa_sk:slovensky-jazyk|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0025906735751295338, + "qf1_stderr": 0.0025906735751295403, + "rouge1": 5.842773836305639e-05, + "rouge1_stderr": 4.122343590294019e-05, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 5.842773836305639e-05, + "rougeL_stderr": 4.122343590294019e-05 + }, + "e_tests.prompt2_qa_sk:dejepis|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "e_tests.prompt3_mqa_single_correct_sk:slovensky-jazyk|0": { + "acc": 0.2205240174672489, + "acc_stderr": 0.019394170839568875, + "em_relaxed": 0.22270742358078602, + "em_relaxed_stderr": 0.019462629121054217 + }, + "mmlu.prompt3_mqa_single_correct_sk:astronomy|0": { + "acc": 0.23026315789473684, + "acc_stderr": 0.03426059424403165, + "em_relaxed": 0.23684210526315788, + "em_relaxed_stderr": 0.03459777606810535 + }, + "mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0": { + "acc": 0.2702702702702703, + "acc_stderr": 0.02764841916772191, + "em_relaxed": 0.28185328185328185, + "em_relaxed_stderr": 0.028009688894113486 + }, + "mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0": { + "acc": 0.21100917431192662, + "acc_stderr": 0.039262212380368526, + "em_relaxed": 0.22018348623853212, + "em_relaxed_stderr": 0.03987279758533731 + }, + "e_tests.prompt1_mqa_multiple_correct_sk:chemia|0": { + "acc": 0.045454545454545456, + "acc_stderr": 0.031765283723714 + }, + "slovak_tests.prompt3_mqa_single_correct_sk:monitor9|0": { + "acc": 0.19815668202764977, + "acc_stderr": 0.02712203315362448, + "em_relaxed": 0.19815668202764977, + "em_relaxed_stderr": 0.02712203315362448 + }, + "mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0": { + "acc": 0.2092426187419769, + "acc_stderr": 0.014583336673745648, + "em_relaxed": 0.21181001283697048, + "em_relaxed_stderr": 0.014648693814173547 + }, + "e_tests.prompt3_mqa_multiple_correct_sk:informatika|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "e_tests.prompt2_mqa_single_correct_sk:biologia|0": { + "acc": 0.14942528735632185, + "acc_stderr": 0.03844315810763212, + "em_relaxed": 0.14942528735632185, + "em_relaxed_stderr": 0.03844315810763212 + }, + "mmlu.prompt1_mqa_single_correct_sk:virology|0": { + "acc": 0.23780487804878048, + "acc_stderr": 0.03334645408665337, + "em_relaxed": 0.24390243902439024, + "em_relaxed_stderr": 0.03363591048272823 + }, + "e_tests.prompt1_mqa_multiple_correct_sk:obcianska-nauka|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "mmlu.prompt2_mqa_single_correct_sk:human_aging|0": { + "acc": 0.1232876712328767, + "acc_stderr": 0.022266919281488138, + "em_relaxed": 0.1232876712328767, + "em_relaxed_stderr": 0.022266919281488138 + }, + "e_tests.prompt1_qa_sk:geografia|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "sklep.prompt3_nli_3_classes_sk:nli|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "scio.prompt3_mqa_single_correct_sk:vsp|0": { + "acc": 0.20388349514563106, + "acc_stderr": 0.014043631695334572, + "em_relaxed": 0.21723300970873785, + "em_relaxed_stderr": 0.014374053054620224 + }, + "mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0": { + "acc": 0.2848101265822785, + "acc_stderr": 0.03601957344556447, + "em_relaxed": 0.2848101265822785, + "em_relaxed_stderr": 0.03601957344556447 + }, + "sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "arc.prompt1_mqa_single_correct_sk|0": { + "acc": 0.2728828058169376, + "acc_stderr": 0.013033734552785697, + "em_relaxed": 0.2728828058169376, + "em_relaxed_stderr": 0.013033734552785697 + }, + "mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0": { + "acc": 0.18181818181818182, + "acc_stderr": 0.038961038961039, + "em_relaxed": 0.18181818181818182, + "em_relaxed_stderr": 0.038961038961039 + }, + "mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0": { + "acc": 0.2616822429906542, + "acc_stderr": 0.0426929191572811, + "em_relaxed": 0.27102803738317754, + "em_relaxed_stderr": 0.04317273776566666 + }, + "belebele.prompt2_mqa_single_correct_sk|0": { + "acc": 0.2511111111111111, + "acc_stderr": 0.014463114105170805, + "em_relaxed": 0.25666666666666665, + "em_relaxed_stderr": 0.01456789134238002 + }, + "mmlu.prompt2_mqa_single_correct_sk:professional_law|0": { + "acc": 0.20922288642186165, + "acc_stderr": 0.0118915566619952, + "em_relaxed": 0.21178479931682323, + "em_relaxed_stderr": 0.011944744511660903 + }, + "senti_sk.prompt2_sentiment_3_class_sk|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0": { + "acc": 0.22018348623853212, + "acc_stderr": 0.03987279758533728, + "em_relaxed": 0.22018348623853212, + "em_relaxed_stderr": 0.03987279758533728 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0": { + "acc": 0.13333333333333333, + "acc_stderr": 0.029365879728106854, + "em_relaxed": 0.13333333333333333, + "em_relaxed_stderr": 0.029365879728106854 + }, + "e_tests.prompt1_mqa_multiple_correct_sk:slovensky-jazyk|0": { + "acc": 0.014285714285714285, + "acc_stderr": 0.010065123433632259 + }, + "sklep.prompt2_hate_speech_sk:hate-speech|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slovakcopa.prompt3_copa_sk|0": { + "acc": 0.144, + "acc_stderr": 0.015716934945725767, + "em_relaxed": 0.144, + "em_relaxed_stderr": 0.015716934945725767 + }, + "mmlu.prompt1_mqa_single_correct_sk:prehistory|0": { + "acc": 0.2852664576802508, + "acc_stderr": 0.025321176274679096, + "em_relaxed": 0.2852664576802508, + "em_relaxed_stderr": 0.025321176274679096 + }, + "mmlu.prompt2_mqa_single_correct_sk:public_relations|0": { + "acc": 0.1651376146788991, + "acc_stderr": 0.0357288117600854, + "em_relaxed": 0.1651376146788991, + "em_relaxed_stderr": 0.0357288117600854 + }, + "e_tests.prompt3_mqa_multiple_correct_sk:obcianska-nauka|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "mmlu.prompt1_mqa_single_correct_sk:anatomy|0": { + "acc": 0.3656716417910448, + "acc_stderr": 0.041761576413228745, + "em_relaxed": 0.373134328358209, + "em_relaxed_stderr": 0.041936678130523646 + }, + "slovak_tests.prompt1_mqa_single_correct_sk:prijmacie_skusky|0": { + "acc": 0.21683673469387754, + "acc_stderr": 0.02084031144263498, + "em_relaxed": 0.2193877551020408, + "em_relaxed_stderr": 0.020928374259338522 + }, + "globalmmlu-lite.prompt1_mqa_single_correct_sk|0": { + "acc": 0.24, + "acc_stderr": 0.021380899352993928, + "em_relaxed": 0.2425, + "em_relaxed_stderr": 0.021456591955524264 + }, + "mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0": { + "acc": 0.30638297872340425, + "acc_stderr": 0.03013590647851756, + "em_relaxed": 0.3148936170212766, + "em_relaxed_stderr": 0.03036358219723817 + }, + "mmlu.prompt1_mqa_single_correct_sk:philosophy|0": { + "acc": 0.22402597402597402, + "acc_stderr": 0.023795979800861743, + "em_relaxed": 0.22402597402597402, + "em_relaxed_stderr": 0.023795979800861743 + }, + "mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0": { + "acc": 0.2911392405063291, + "acc_stderr": 0.03625609398754825, + "em_relaxed": 0.2974683544303797, + "em_relaxed_stderr": 0.03648408889243667 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0": { + "acc": 0.19689119170984457, + "acc_stderr": 0.028697873971860688, + "em_relaxed": 0.19689119170984457, + "em_relaxed_stderr": 0.028697873971860688 + }, + "scio.prompt3_mqa_single_correct_sk:mat|0": { + "acc": 0.20765027322404372, + "acc_stderr": 0.021231382642567846, + "em_relaxed": 0.21584699453551912, + "em_relaxed_stderr": 0.02153411244027524 + }, + "e_tests.prompt2_qa_sk:geografia|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "mmlu.prompt1_mqa_single_correct_sk:public_relations|0": { + "acc": 0.21100917431192662, + "acc_stderr": 0.03926221238036852, + "em_relaxed": 0.21100917431192662, + "em_relaxed_stderr": 0.03926221238036852 + }, + "mmlu.prompt2_mqa_single_correct_sk:marketing|0": { + "acc": 0.17094017094017094, + "acc_stderr": 0.02466249684520983, + "em_relaxed": 0.17094017094017094, + "em_relaxed_stderr": 0.02466249684520983 + }, + "scio.prompt1_mqa_single_correct_sk:bio|0": { + "acc": 0.2743362831858407, + "acc_stderr": 0.024268949584718702, + "em_relaxed": 0.28023598820059, + "em_relaxed_stderr": 0.024428605014452955 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0": { + "acc": 0.24352331606217617, + "acc_stderr": 0.03097543638684542, + "em_relaxed": 0.24870466321243523, + "em_relaxed_stderr": 0.0311958408777003 + }, + "sklep.prompt3_qa_sk:question-answering|0": { + "qem": 0.0002087029114056141, + "qem_stderr": 0.0001475675430580104, + "qf1": 0.006093522810123945, + "qf1_stderr": 0.00032767921496533014 + }, + "e_tests.prompt3_qa_sk:biologia|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0011204481792717084, + "rouge1_stderr": 0.0011204481792717086, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0011204481792717084, + "rougeL_stderr": 0.0011204481792717086 + }, + "slovak_tests.prompt3_mqa_single_correct_sk:vyssie_studium|0": { + "acc": 0.13333333333333333, + "acc_stderr": 0.03333333333333327, + "em_relaxed": 0.13333333333333333, + "em_relaxed_stderr": 0.03333333333333327 + }, + "mmlu.prompt3_mqa_single_correct_sk:anatomy|0": { + "acc": 0.23134328358208955, + "acc_stderr": 0.03656527469759901, + "em_relaxed": 0.23134328358208955, + "em_relaxed_stderr": 0.03656527469759901 + }, + "mmlu.prompt1_mqa_single_correct_sk:sociology|0": { + "acc": 0.2182741116751269, + "acc_stderr": 0.029505319811543015, + "em_relaxed": 0.2182741116751269, + "em_relaxed_stderr": 0.029505319811543015 + }, + "e_tests.prompt2_mqa_single_correct_sk:matematika|0": { + "acc": 0.10256410256410256, + "acc_stderr": 0.04921614717627658, + "em_relaxed": 0.3076923076923077, + "em_relaxed_stderr": 0.074871425137275 + }, + "scio.prompt1_mqa_single_correct_sk:aj|0": { + "acc": 0.6, + "acc_stderr": 0.16329931618554522, + "em_relaxed": 0.6, + "em_relaxed_stderr": 0.16329931618554522 + }, + "mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0": { + "acc": 0.22393822393822393, + "acc_stderr": 0.02595388693967799, + "em_relaxed": 0.23166023166023167, + "em_relaxed_stderr": 0.026265915716011466 + }, + "mmlu.prompt2_mqa_single_correct_sk:global_facts|0": { + "acc": 0.2, + "acc_stderr": 0.04020151261036846, + "em_relaxed": 0.2, + "em_relaxed_stderr": 0.04020151261036846 + }, + "financial_exams.prompt1_completion_sk|0": { + "acc": 0.053973013493253376, + "acc_stderr": 0.006189064679835392, + "em_relaxed": 0.05697151424287856, + "em_relaxed_stderr": 0.006348574383669881 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0": { + "acc": 0.28110599078341014, + "acc_stderr": 0.03058726229145915, + "em_relaxed": 0.2903225806451613, + "em_relaxed_stderr": 0.030884745411494743 + }, + "slovak_tests.prompt3_qa_sk:prijmacie_skusky|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0012698412698412698, + "rouge1_stderr": 0.00126984126984127, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0012698412698412698, + "rougeL_stderr": 0.00126984126984127 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0": { + "acc": 0.22807017543859648, + "acc_stderr": 0.056069817847611766, + "em_relaxed": 0.22807017543859648, + "em_relaxed_stderr": 0.056069817847611766 + }, + "mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0": { + "acc": 0.2753623188405797, + "acc_stderr": 0.02693682252998795, + "em_relaxed": 0.27898550724637683, + "em_relaxed_stderr": 0.027045590734089176 + }, + "ifeval_sk.ifeval|0": { + "inst_level_loose_acc": 0.17312348668280872, + "inst_level_loose_acc_stderr": 0.0004210039776560394, + "inst_level_strict_acc": 0.17312348668280872, + "inst_level_strict_acc_stderr": 0.0004210039776560394, + "prompt_level_loose_acc": 0.0782122905027933, + "prompt_level_loose_acc_stderr": 0.011597665111537078, + "prompt_level_strict_acc": 0.0782122905027933, + "prompt_level_strict_acc_stderr": 0.011597665111537078 + }, + "mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0": { + "acc": 0.2826086956521739, + "acc_stderr": 0.027152165210427823, + "em_relaxed": 0.286231884057971, + "em_relaxed_stderr": 0.02725657169175206 + }, + "mmlu.prompt3_mqa_single_correct_sk:professional_law|0": { + "acc": 0.2314261315115286, + "acc_stderr": 0.012329801197790603, + "em_relaxed": 0.23825789923142612, + "em_relaxed_stderr": 0.012454741062931133 + }, + "mmlu.prompt3_mqa_single_correct_sk:public_relations|0": { + "acc": 0.1834862385321101, + "acc_stderr": 0.03724531275523975, + "em_relaxed": 0.1834862385321101, + "em_relaxed_stderr": 0.03724531275523975 + }, + "movie_review.prompt1_sentiment_binary_sk|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slovak_tests.prompt1_qa_sk:monitor9|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0": { + "acc": 0.2416555407209613, + "acc_stderr": 0.01565239642514941, + "em_relaxed": 0.2416555407209613, + "em_relaxed_stderr": 0.01565239642514941 + }, + "slovak_tests.prompt1_mqa_single_correct_sk:maturita|0": { + "acc": 0.15718157181571815, + "acc_stderr": 0.018973345526861658, + "em_relaxed": 0.16260162601626016, + "em_relaxed_stderr": 0.019235549151947248 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0": { + "acc": 0.22897196261682243, + "acc_stderr": 0.028789653442089263, + "em_relaxed": 0.24766355140186916, + "em_relaxed_stderr": 0.02957653529316449 + }, + "mmlu.prompt3_mqa_single_correct_sk:formal_logic|0": { + "acc": 0.17391304347826086, + "acc_stderr": 0.03549985134468377, + "em_relaxed": 0.20869565217391303, + "em_relaxed_stderr": 0.03806063686277675 + }, + "mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0": { + "acc": 0.21983914209115282, + "acc_stderr": 0.02147203067557281, + "em_relaxed": 0.21983914209115282, + "em_relaxed_stderr": 0.02147203067557281 + }, + "toxic_sk.prompt1_toxicity_sk|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "driving_test.prompt2_mqa_single_correct_sk|0": { + "acc": 0.09985734664764621, + "acc_stderr": 0.011331741573886145, + "em_relaxed": 0.10271041369472182, + "em_relaxed_stderr": 0.011474255943096628 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0": { + "acc": 0.32323232323232326, + "acc_stderr": 0.04724590344515123, + "em_relaxed": 0.3333333333333333, + "em_relaxed_stderr": 0.04761904761904759 + }, + "mmlu.prompt2_mqa_single_correct_sk:astronomy|0": { + "acc": 0.19736842105263158, + "acc_stderr": 0.03238981601699397, + "em_relaxed": 0.21052631578947367, + "em_relaxed_stderr": 0.033176727875331574 + }, + "mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0": { + "acc": 0.2803030303030303, + "acc_stderr": 0.027695605168511307, + "em_relaxed": 0.2878787878787879, + "em_relaxed_stderr": 0.027919261062611087 + }, + "mmlu.prompt1_mqa_single_correct_sk:professional_law|0": { + "acc": 0.2322801024765158, + "acc_stderr": 0.012345664537818761, + "em_relaxed": 0.23654995730145176, + "em_relaxed_stderr": 0.012423924913578318 + }, + "sklep.prompt2_nli_3_classes_sk:nli|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0": { + "acc": 0.2119815668202765, + "acc_stderr": 0.02780931949344348, + "em_relaxed": 0.22580645161290322, + "em_relaxed_stderr": 0.028448938828651484 + }, + "sklep.prompt1_nli_binary_sk:rte|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "e_tests.prompt3_qa_sk:dejepis|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "mmlu.prompt3_mqa_single_correct_sk:computer_security|0": { + "acc": 0.22, + "acc_stderr": 0.0416333199893227, + "em_relaxed": 0.24, + "em_relaxed_stderr": 0.04292346959909283 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0": { + "acc": 0.2631578947368421, + "acc_stderr": 0.05884389414473131, + "em_relaxed": 0.2807017543859649, + "em_relaxed_stderr": 0.06004585739704729 + }, + "slovak_tests.prompt2_mqa_single_correct_sk:prijmacie_skusky|0": { + "acc": 0.13520408163265307, + "acc_stderr": 0.017292721140889967, + "em_relaxed": 0.1377551020408163, + "em_relaxed_stderr": 0.01742933367587574 + }, + "scio.prompt3_mqa_single_correct_sk:che|0": { + "acc": 0.22764227642276422, + "acc_stderr": 0.026788763286110986, + "em_relaxed": 0.23577235772357724, + "em_relaxed_stderr": 0.027119068752977968 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0": { + "acc": 0.23333333333333334, + "acc_stderr": 0.03464962582372412, + "em_relaxed": 0.23333333333333334, + "em_relaxed_stderr": 0.03464962582372412 + }, + "mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0": { + "acc": 0.1919191919191919, + "acc_stderr": 0.03978080447933685, + "em_relaxed": 0.1919191919191919, + "em_relaxed_stderr": 0.03978080447933685 + }, + "mmlu.prompt2_mqa_single_correct_sk:econometrics|0": { + "acc": 0.16071428571428573, + "acc_stderr": 0.034859460964757394, + "em_relaxed": 0.16071428571428573, + "em_relaxed_stderr": 0.034859460964757394 + }, + "mmlu.prompt1_mqa_single_correct_sk:college_biology|0": { + "acc": 0.2535211267605634, + "acc_stderr": 0.03663585085277138, + "em_relaxed": 0.2605633802816901, + "em_relaxed_stderr": 0.03696558732615873 + }, + "mmlu.prompt3_mqa_single_correct_sk:nutrition|0": { + "acc": 0.22442244224422442, + "acc_stderr": 0.024007242951044432, + "em_relaxed": 0.25412541254125415, + "em_relaxed_stderr": 0.02505263910183634 + }, + "slovak_tests.prompt1_qa_sk:prijmacie_skusky|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.001330671989354624, + "rouge1_stderr": 0.0013306719893546237, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.001330671989354624, + "rougeL_stderr": 0.0013306719893546237 + }, + "mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0": { + "acc": 0.18711656441717792, + "acc_stderr": 0.021633588964875924, + "em_relaxed": 0.19325153374233128, + "em_relaxed_stderr": 0.02190225688747218 + }, + "e_tests.prompt2_mqa_single_correct_sk:slovensky-jazyk|0": { + "acc": 0.21615720524017468, + "acc_stderr": 0.0192548989397317, + "em_relaxed": 0.2183406113537118, + "em_relaxed_stderr": 0.01932493025894029 + }, + "mmlu.prompt2_mqa_single_correct_sk:business_ethics|0": { + "acc": 0.14, + "acc_stderr": 0.0348735088019777, + "em_relaxed": 0.16, + "em_relaxed_stderr": 0.03684529491774708 + }, + "e_tests.prompt1_mqa_single_correct_sk:geografia|0": { + "acc": 0.20408163265306123, + "acc_stderr": 0.058172215566282555, + "em_relaxed": 0.20408163265306123, + "em_relaxed_stderr": 0.058172215566282555 + }, + "e_tests.prompt2_qa_sk:fyzika|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "multi_wiki_qa.prompt3_qa_sk|0": { + "qem": 0.0006003602161296778, + "qem_stderr": 0.00034654874631857635, + "qf1": 0.005765729879562343, + "qf1_stderr": 0.0005521722468476828 + }, + "e_tests.prompt1_qa_sk:fyzika|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0": { + "acc": 0.19172932330827067, + "acc_stderr": 0.024182419147367247, + "em_relaxed": 0.20676691729323307, + "em_relaxed_stderr": 0.024878145232561027 + }, + "e_tests.prompt2_mqa_single_correct_sk:obcianska-nauka|0": { + "acc": 0.15942028985507245, + "acc_stderr": 0.04439221265797129, + "em_relaxed": 0.15942028985507245, + "em_relaxed_stderr": 0.04439221265797129 + }, + "e_tests.prompt2_mqa_single_correct_sk:dejepis|0": { + "acc": 0.20833333333333334, + "acc_stderr": 0.059238169759886904, + "em_relaxed": 0.20833333333333334, + "em_relaxed_stderr": 0.059238169759886904 + }, + "scio.prompt1_mqa_single_correct_sk:vsp|0": { + "acc": 0.24150485436893204, + "acc_stderr": 0.01491898981873877, + "em_relaxed": 0.25849514563106796, + "em_relaxed_stderr": 0.015261010021381633 + }, + "mmlu.prompt3_mqa_single_correct_sk:business_ethics|0": { + "acc": 0.16, + "acc_stderr": 0.03684529491774709, + "em_relaxed": 0.17, + "em_relaxed_stderr": 0.0377525168068637 + }, + "mmlu.prompt3_mqa_single_correct_sk:international_law|0": { + "acc": 0.3050847457627119, + "acc_stderr": 0.04256799926288002, + "em_relaxed": 0.3050847457627119, + "em_relaxed_stderr": 0.04256799926288002 + }, + "mmlu.prompt1_mqa_single_correct_sk:international_law|0": { + "acc": 0.288135593220339, + "acc_stderr": 0.04187011593049808, + "em_relaxed": 0.2966101694915254, + "em_relaxed_stderr": 0.04222776832233627 + }, + "slovakcopa.prompt2_copa_sk|0": { + "acc": 0.006, + "acc_stderr": 0.003457152557758384, + "em_relaxed": 0.006, + "em_relaxed_stderr": 0.003457152557758384 + }, + "mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0": { + "acc": 0.2116564417177914, + "acc_stderr": 0.022658534978587233, + "em_relaxed": 0.22085889570552147, + "em_relaxed_stderr": 0.023010381863796215 + }, + "e_tests.prompt2_mqa_multiple_correct_sk:fyzika|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "mmlu.prompt3_mqa_single_correct_sk:global_facts|0": { + "acc": 0.24, + "acc_stderr": 0.042923469599092816, + "em_relaxed": 0.25, + "em_relaxed_stderr": 0.04351941398892446 + }, + "mmlu.prompt3_mqa_single_correct_sk:world_religions|0": { + "acc": 0.2807017543859649, + "acc_stderr": 0.034462962170884265, + "em_relaxed": 0.2807017543859649, + "em_relaxed_stderr": 0.034462962170884265 + }, + "e_tests.prompt1_qa_sk:biologia|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0": { + "acc": 0.15, + "acc_stderr": 0.03588702812826371, + "em_relaxed": 0.15, + "em_relaxed_stderr": 0.03588702812826371 + }, + "e_tests.prompt3_mqa_single_correct_sk:chemia|0": { + "acc": 0.07142857142857142, + "acc_stderr": 0.03472660248602842, + "em_relaxed": 0.08928571428571429, + "em_relaxed_stderr": 0.038450387280282494 + }, + "e_tests.prompt1_mqa_single_correct_sk:matematika|0": { + "acc": 0.15384615384615385, + "acc_stderr": 0.05852968540405249, + "em_relaxed": 0.358974358974359, + "em_relaxed_stderr": 0.07781756136754926 + }, + "e_tests.prompt3_qa_sk:geografia|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "e_tests.prompt2_mqa_multiple_correct_sk:chemia|0": { + "acc": 0.045454545454545456, + "acc_stderr": 0.031765283723714 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0": { + "acc": 0.2383177570093458, + "acc_stderr": 0.029192770642101563, + "em_relaxed": 0.2570093457943925, + "em_relaxed_stderr": 0.029941691533244646 + }, + "mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0": { + "acc": 0.24932975871313673, + "acc_stderr": 0.022430563714050267, + "em_relaxed": 0.24932975871313673, + "em_relaxed_stderr": 0.022430563714050267 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0": { + "acc": 0.24468085106382978, + "acc_stderr": 0.025645553622266722, + "em_relaxed": 0.25177304964539005, + "em_relaxed_stderr": 0.025892151156709405 + }, + "e_tests.prompt1_mqa_multiple_correct_sk:informatika|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "mmlu.prompt2_mqa_single_correct_sk:college_biology|0": { + "acc": 0.22535211267605634, + "acc_stderr": 0.035186286213623755, + "em_relaxed": 0.22535211267605634, + "em_relaxed_stderr": 0.035186286213623755 + }, + "mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0": { + "acc": 0.296551724137931, + "acc_stderr": 0.03806142687309993, + "em_relaxed": 0.3103448275862069, + "em_relaxed_stderr": 0.03855289616378948 + }, + "e_tests.prompt1_mqa_single_correct_sk:chemia|0": { + "acc": 0.10714285714285714, + "acc_stderr": 0.0417053005800816, + "em_relaxed": 0.10714285714285714, + "em_relaxed_stderr": 0.0417053005800816 + }, + "mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0": { + "acc": 0.25671140939597314, + "acc_stderr": 0.01790782476183757, + "em_relaxed": 0.25838926174496646, + "em_relaxed_stderr": 0.017945962358551722 + }, + "e_tests.prompt3_mqa_multiple_correct_sk:chemia|0": { + "acc": 0.045454545454545456, + "acc_stderr": 0.031765283723714 + }, + "e_tests.prompt3_qa_sk:obcianska-nauka|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "mmlu.prompt2_mqa_single_correct_sk:computer_security|0": { + "acc": 0.2, + "acc_stderr": 0.04020151261036843, + "em_relaxed": 0.22, + "em_relaxed_stderr": 0.041633319989322716 + }, + "mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0": { + "acc": 0.2018348623853211, + "acc_stderr": 0.03862180715692686, + "em_relaxed": 0.2018348623853211, + "em_relaxed_stderr": 0.03862180715692686 + }, + "e_tests.prompt3_mqa_single_correct_sk:obcianska-nauka|0": { + "acc": 0.14492753623188406, + "acc_stderr": 0.0426896366873121, + "em_relaxed": 0.14492753623188406, + "em_relaxed_stderr": 0.0426896366873121 + }, + "e_tests.prompt3_mqa_single_correct_sk:fyzika|0": { + "acc": 0.18181818181818182, + "acc_stderr": 0.08416546361568647, + "em_relaxed": 0.2727272727272727, + "em_relaxed_stderr": 0.0971859061499725 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0": { + "acc": 0.16917293233082706, + "acc_stderr": 0.023030205866522328, + "em_relaxed": 0.17669172932330826, + "em_relaxed_stderr": 0.023429683004809066 + }, + "e_tests.prompt2_mqa_multiple_correct_sk:dejepis|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "e_tests.prompt2_mqa_multiple_correct_sk:geografia|0": { + "acc": 0.034482758620689655, + "acc_stderr": 0.034482758620689676 + }, + "e_tests.prompt3_qa_sk:chemia|0": { + "qem": 0.018518518518518517, + "qem_stderr": 0.018518518518518517, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0006660882976672449, + "rouge1_stderr": 0.0004499203448417917, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0006660882976672449, + "rougeL_stderr": 0.0004499203448417917 + }, + "e_tests.prompt3_mqa_multiple_correct_sk:biologia|0": { + "acc": 0.03571428571428571, + "acc_stderr": 0.02502318034804895 + }, + "mmlu.prompt1_mqa_single_correct_sk:formal_logic|0": { + "acc": 0.1826086956521739, + "acc_stderr": 0.03618456100987082, + "em_relaxed": 0.21739130434782608, + "em_relaxed_stderr": 0.038631448549506 + }, + "mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0": { + "acc": 0.21379310344827587, + "acc_stderr": 0.034165204477475494, + "em_relaxed": 0.2206896551724138, + "em_relaxed_stderr": 0.03455930201924811 + }, + "e_tests.prompt2_mqa_multiple_correct_sk:slovensky-jazyk|0": { + "acc": 0.014285714285714285, + "acc_stderr": 0.010065123433632259 + }, + "e_tests.prompt1_mqa_multiple_correct_sk:matematika|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0": { + "acc": 0.22, + "acc_stderr": 0.041633319989322695, + "em_relaxed": 0.24, + "em_relaxed_stderr": 0.042923469599092816 + }, + "mmlu.prompt1_mqa_single_correct_sk:marketing|0": { + "acc": 0.24786324786324787, + "acc_stderr": 0.028286324075564397, + "em_relaxed": 0.25213675213675213, + "em_relaxed_stderr": 0.02844796547623102 + }, + "mmlu.prompt1_mqa_single_correct_sk:world_religions|0": { + "acc": 0.26900584795321636, + "acc_stderr": 0.0340105262010409, + "em_relaxed": 0.27485380116959063, + "em_relaxed_stderr": 0.034240429246915824 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0": { + "acc": 0.21717171717171718, + "acc_stderr": 0.02937661648494564, + "em_relaxed": 0.21717171717171718, + "em_relaxed_stderr": 0.02937661648494564 + }, + "mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0": { + "acc": 0.26702269692923897, + "acc_stderr": 0.01617590372442817, + "em_relaxed": 0.26702269692923897, + "em_relaxed_stderr": 0.01617590372442817 + }, + "e_tests.prompt2_qa_sk:informatika|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.010073260073260074, + "qf1_stderr": 0.007341600122632042, + "rouge1": 0.0012475950143262986, + "rouge1_stderr": 0.000686304468853021, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0012475950143262986, + "rougeL_stderr": 0.000686304468853021 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0": { + "acc": 0.2474747474747475, + "acc_stderr": 0.030746300742124498, + "em_relaxed": 0.25252525252525254, + "em_relaxed_stderr": 0.030954055470365897 + }, + "mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0": { + "acc": 0.24242424242424243, + "acc_stderr": 0.04329004329004326, + "em_relaxed": 0.26262626262626265, + "em_relaxed_stderr": 0.04445287676983944 + }, + "mmlu.prompt1_mqa_single_correct_sk:machine_learning|0": { + "acc": 0.16964285714285715, + "acc_stderr": 0.0356236785009539, + "em_relaxed": 0.19642857142857142, + "em_relaxed_stderr": 0.03770970049347018 + }, + "mmlu.prompt3_mqa_single_correct_sk:virology|0": { + "acc": 0.22560975609756098, + "acc_stderr": 0.03273897454566342, + "em_relaxed": 0.22560975609756098, + "em_relaxed_stderr": 0.03273897454566342 + }, + "slovakcopa.prompt1_copa_sk|0": { + "acc": 0.01, + "acc_stderr": 0.0044541754970721, + "em_relaxed": 0.01, + "em_relaxed_stderr": 0.0044541754970721 + }, + "scio.prompt3_mqa_single_correct_sk:bio|0": { + "acc": 0.3008849557522124, + "acc_stderr": 0.024946878423156852, + "em_relaxed": 0.30973451327433627, + "em_relaxed_stderr": 0.025150378763664 + }, + "mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0": { + "acc": 0.205607476635514, + "acc_stderr": 0.03925401580070484, + "em_relaxed": 0.21495327102803738, + "em_relaxed_stderr": 0.03989944463395407 + }, + "multi_wiki_qa.prompt2_qa_sk|0": { + "qem": 0.0006003602161296778, + "qem_stderr": 0.0003465487463185779, + "qf1": 0.005417769336679904, + "qf1_stderr": 0.0005509525930025259 + }, + "mmlu.prompt3_mqa_single_correct_sk:college_physics|0": { + "acc": 0.21568627450980393, + "acc_stderr": 0.04092563958237654, + "em_relaxed": 0.22549019607843138, + "em_relaxed_stderr": 0.04158307533083286 + }, + "mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0": { + "acc": 0.20425531914893616, + "acc_stderr": 0.026355158413349424, + "em_relaxed": 0.20425531914893616, + "em_relaxed_stderr": 0.026355158413349424 + }, + "e_tests.prompt1_mqa_single_correct_sk:obcianska-nauka|0": { + "acc": 0.18840579710144928, + "acc_stderr": 0.047420064740574204, + "em_relaxed": 0.18840579710144928, + "em_relaxed_stderr": 0.047420064740574204 + }, + "e_tests.prompt3_mqa_single_correct_sk:dejepis|0": { + "acc": 0.20833333333333334, + "acc_stderr": 0.059238169759886904, + "em_relaxed": 0.20833333333333334, + "em_relaxed_stderr": 0.059238169759886904 + }, + "mmlu.prompt2_mqa_single_correct_sk:prehistory|0": { + "acc": 0.19122257053291536, + "acc_stderr": 0.022053144718757862, + "em_relaxed": 0.19122257053291536, + "em_relaxed_stderr": 0.022053144718757862 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0": { + "acc": 0.23333333333333334, + "acc_stderr": 0.03464962582372412, + "em_relaxed": 0.23333333333333334, + "em_relaxed_stderr": 0.03464962582372412 + }, + "sklep.prompt1_hate_speech_sk:hate-speech|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "driving_test.prompt1_mqa_single_correct_sk|0": { + "acc": 0.16405135520684735, + "acc_stderr": 0.01399686230459443, + "em_relaxed": 0.16833095577746077, + "em_relaxed_stderr": 0.014141915666582279 + }, + "e_tests.prompt3_mqa_single_correct_sk:informatika|0": { + "acc": 0.256, + "acc_stderr": 0.03919183588453089, + "em_relaxed": 0.264, + "em_relaxed_stderr": 0.03958494337416042 + }, + "mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0": { + "acc": 0.18959731543624161, + "acc_stderr": 0.01606971018943585, + "em_relaxed": 0.19463087248322147, + "em_relaxed_stderr": 0.0162309847870997 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0": { + "acc": 0.22429906542056074, + "acc_stderr": 0.028580583273338626, + "em_relaxed": 0.2383177570093458, + "em_relaxed_stderr": 0.029192770642101563 + }, + "e_tests.prompt1_mqa_single_correct_sk:fyzika|0": { + "acc": 0.13636363636363635, + "acc_stderr": 0.07488677009526491, + "em_relaxed": 0.22727272727272727, + "em_relaxed_stderr": 0.09144861547306321 + }, + "mmlu.prompt1_mqa_single_correct_sk:college_medicine|0": { + "acc": 0.26744186046511625, + "acc_stderr": 0.03384836428157859, + "em_relaxed": 0.27906976744186046, + "em_relaxed_stderr": 0.03430085607014882 + }, + "slovak_tests.prompt3_qa_sk:monitor9|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "mmlu.prompt3_mqa_single_correct_sk:human_aging|0": { + "acc": 0.136986301369863, + "acc_stderr": 0.023287301596102452, + "em_relaxed": 0.136986301369863, + "em_relaxed_stderr": 0.023287301596102452 + }, + "globalmmlu-lite.prompt3_mqa_single_correct_sk|0": { + "acc": 0.2025, + "acc_stderr": 0.0201183061066658, + "em_relaxed": 0.205, + "em_relaxed_stderr": 0.02021035988339997 + }, + "sklep.prompt2_qa_sk:question-answering|0": { + "qem": 0.0002087029114056141, + "qem_stderr": 0.0001475675430580104, + "qf1": 0.006743145308527306, + "qf1_stderr": 0.00039734127192685853 + }, + "truthfulqa.prompt2_mqa_single_correct_sk:mc1|0": { + "acc": 0.04884318766066838, + "acc_stderr": 0.00773245768018304, + "em_relaxed": 0.08354755784061697, + "em_relaxed_stderr": 0.009926847196525902 + }, + "senti_sk.prompt3_sentiment_binary_sk|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "scio.prompt2_mqa_single_correct_sk:vsp|0": { + "acc": 0.23786407766990292, + "acc_stderr": 0.014841600192116697, + "em_relaxed": 0.2512135922330097, + "em_relaxed_stderr": 0.015118218765838777 + }, + "e_tests.prompt3_mqa_multiple_correct_sk:matematika|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "e_tests.prompt3_qa_sk:slovensky-jazyk|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0025906735751295338, + "qf1_stderr": 0.0025906735751295403, + "rouge1": 0.00035708568846413394, + "rouge1_stderr": 0.00020841637674444064, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.00035708568846413394, + "rougeL_stderr": 0.00020841637674444064 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0": { + "acc": 0.24561403508771928, + "acc_stderr": 0.05752139056405265, + "em_relaxed": 0.2631578947368421, + "em_relaxed_stderr": 0.05884389414473131 + }, + "mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0": { + "acc": 0.16595744680851063, + "acc_stderr": 0.02432117475103868, + "em_relaxed": 0.16595744680851063, + "em_relaxed_stderr": 0.02432117475103868 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0": { + "acc": 0.2126865671641791, + "acc_stderr": 0.01769159425053082, + "em_relaxed": 0.21641791044776118, + "em_relaxed_stderr": 0.01780376934180082 + }, + "senti_sk.prompt2_sentiment_binary_sk|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "scio.prompt1_mqa_single_correct_sk:che|0": { + "acc": 0.25609756097560976, + "acc_stderr": 0.027885450835842325, + "em_relaxed": 0.25609756097560976, + "em_relaxed_stderr": 0.027885450835842325 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0": { + "acc": 0.24444444444444444, + "acc_stderr": 0.03212157057535212, + "em_relaxed": 0.25, + "em_relaxed_stderr": 0.032364888900157734 + }, + "mmlu.prompt2_mqa_single_correct_sk:machine_learning|0": { + "acc": 0.14285714285714285, + "acc_stderr": 0.03321361106966269, + "em_relaxed": 0.16071428571428573, + "em_relaxed_stderr": 0.0348594609647574 + }, + "mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0": { + "acc": 0.21212121212121213, + "acc_stderr": 0.04129606932540889, + "em_relaxed": 0.24242424242424243, + "em_relaxed_stderr": 0.04329004329004327 + }, + "mmlu.prompt3_mqa_single_correct_sk:prehistory|0": { + "acc": 0.26959247648902823, + "acc_stderr": 0.024884157592464488, + "em_relaxed": 0.26959247648902823, + "em_relaxed_stderr": 0.024884157592464488 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0": { + "acc": 0.2538860103626943, + "acc_stderr": 0.03141024780565318, + "em_relaxed": 0.2538860103626943, + "em_relaxed_stderr": 0.03141024780565318 + }, + "mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0": { + "acc": 0.21212121212121213, + "acc_stderr": 0.04129606932540889, + "em_relaxed": 0.23232323232323232, + "em_relaxed_stderr": 0.042660160170546885 + }, + "truthfulqa.prompt1_mqa_single_correct_sk:mc1|0": { + "acc": 0.042416452442159386, + "acc_stderr": 0.0072301115841424965, + "em_relaxed": 0.07840616966580977, + "em_relaxed_stderr": 0.009643493430258573 + }, + "belebele.prompt3_mqa_single_correct_sk|0": { + "acc": 0.24666666666666667, + "acc_stderr": 0.014377023375409347, + "em_relaxed": 0.2544444444444444, + "em_relaxed_stderr": 0.014526354751055178 + }, + "arc.prompt3_mqa_single_correct_sk|0": { + "acc": 0.22840034217279725, + "acc_stderr": 0.012283523666042804, + "em_relaxed": 0.22840034217279725, + "em_relaxed_stderr": 0.012283523666042804 + }, + "financial_exams.prompt2_completion_sk|0": { + "acc": 0.07871064467766117, + "acc_stderr": 0.00737563958812383, + "em_relaxed": 0.07946026986506746, + "em_relaxed_stderr": 0.007407662917614668 + }, + "e_tests.prompt2_mqa_multiple_correct_sk:matematika|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0": { + "acc": 0.23049645390070922, + "acc_stderr": 0.025123739226872395, + "em_relaxed": 0.24113475177304963, + "em_relaxed_stderr": 0.02551873104953778 + }, + "mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0": { + "acc": 0.24, + "acc_stderr": 0.042923469599092816, + "em_relaxed": 0.24, + "em_relaxed_stderr": 0.042923469599092816 + }, + "e_tests.prompt3_mqa_single_correct_sk:biologia|0": { + "acc": 0.10344827586206896, + "acc_stderr": 0.03283976780985287, + "em_relaxed": 0.11494252873563218, + "em_relaxed_stderr": 0.03439354046699091 + }, + "mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0": { + "acc": 0.24299065420560748, + "acc_stderr": 0.041657429989652724, + "em_relaxed": 0.24299065420560748, + "em_relaxed_stderr": 0.041657429989652724 + }, + "mmlu.prompt1_mqa_single_correct_sk:human_aging|0": { + "acc": 0.2191780821917808, + "acc_stderr": 0.028018592326462144, + "em_relaxed": 0.2191780821917808, + "em_relaxed_stderr": 0.028018592326462144 + }, + "slovak_tests.prompt3_mqa_single_correct_sk:maturita|0": { + "acc": 0.18157181571815717, + "acc_stderr": 0.020095119692986566, + "em_relaxed": 0.18157181571815717, + "em_relaxed_stderr": 0.020095119692986566 + }, + "mmlu.prompt2_mqa_single_correct_sk:college_physics|0": { + "acc": 0.22549019607843138, + "acc_stderr": 0.04158307533083286, + "em_relaxed": 0.23529411764705882, + "em_relaxed_stderr": 0.04220773659171455 + }, + "mmlu.prompt1_mqa_single_correct_sk:econometrics|0": { + "acc": 0.20535714285714285, + "acc_stderr": 0.03834241021419071, + "em_relaxed": 0.22321428571428573, + "em_relaxed_stderr": 0.039523019677025116 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0": { + "acc": 0.1921641791044776, + "acc_stderr": 0.01703416618213854, + "em_relaxed": 0.1958955223880597, + "em_relaxed_stderr": 0.01715898536836359 + }, + "mmlu.prompt1_mqa_single_correct_sk:college_physics|0": { + "acc": 0.23529411764705882, + "acc_stderr": 0.04220773659171455, + "em_relaxed": 0.24509803921568626, + "em_relaxed_stderr": 0.042801058373643945 + }, + "e_tests.prompt1_mqa_single_correct_sk:dejepis|0": { + "acc": 0.22916666666666666, + "acc_stderr": 0.06130657275434794, + "em_relaxed": 0.22916666666666666, + "em_relaxed_stderr": 0.06130657275434794 + }, + "scio.prompt2_mqa_single_correct_sk:bio|0": { + "acc": 0.2861356932153392, + "acc_stderr": 0.02458303513041921, + "em_relaxed": 0.2949852507374631, + "em_relaxed_stderr": 0.024805095426679618 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0": { + "acc": 0.23255813953488372, + "acc_stderr": 0.021502792400356754, + "em_relaxed": 0.24031007751937986, + "em_relaxed_stderr": 0.02174755921996615 + }, + "mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0": { + "acc": 0.2537878787878788, + "acc_stderr": 0.02683420376859365, + "em_relaxed": 0.25757575757575757, + "em_relaxed_stderr": 0.026965016533893105 + }, + "toxic_sk.prompt3_toxicity_sk|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0": { + "acc": 0.17777777777777778, + "acc_stderr": 0.033027898599017176, + "em_relaxed": 0.17777777777777778, + "em_relaxed_stderr": 0.033027898599017176 + }, + "driving_test.prompt3_mqa_single_correct_sk|0": { + "acc": 0.14835948644793154, + "acc_stderr": 0.013434972977209091, + "em_relaxed": 0.15121255349500715, + "em_relaxed_stderr": 0.01354080168340582 + }, + "slovak_tests.prompt3_mqa_single_correct_sk:prijmacie_skusky|0": { + "acc": 0.1760204081632653, + "acc_stderr": 0.019259781722629446, + "em_relaxed": 0.1760204081632653, + "em_relaxed_stderr": 0.019259781722629446 + }, + "globalmmlu-lite.prompt2_mqa_single_correct_sk|0": { + "acc": 0.19, + "acc_stderr": 0.019639610121239284, + "em_relaxed": 0.1925, + "em_relaxed_stderr": 0.01973786547054499 + }, + "mmlu.prompt3_mqa_single_correct_sk:philosophy|0": { + "acc": 0.12337662337662338, + "acc_stderr": 0.018769546352956558, + "em_relaxed": 0.12337662337662338, + "em_relaxed_stderr": 0.018769546352956558 + }, + "sklep.prompt1_qa_sk:question-answering|0": { + "qem": 0.0004174058228112282, + "qem_stderr": 0.00020867023775979663, + "qf1": 0.008276210006991061, + "qf1_stderr": 0.00042241089189288785 + }, + "mmlu.prompt1_mqa_single_correct_sk:business_ethics|0": { + "acc": 0.2, + "acc_stderr": 0.04020151261036846, + "em_relaxed": 0.22, + "em_relaxed_stderr": 0.041633319989322674 + }, + "skj1.prompt1_skj1_sk|0": { + "acc": 0.41, + "acc_stderr": 0.04943110704237102, + "em_relaxed": 0.44, + "em_relaxed_stderr": 0.049888765156985884 + }, + "e_tests.prompt2_qa_sk:slovensky-jazyk|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0025906735751295338, + "qf1_stderr": 0.0025906735751295403, + "rouge1": 0.00011405864075384238, + "rouge1_stderr": 8.908193183205394e-05, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.00011405864075384238, + "rougeL_stderr": 8.908193183205394e-05 + }, + "belebele.prompt1_mqa_single_correct_sk|0": { + "acc": 0.2633333333333333, + "acc_stderr": 0.014689553047342515, + "em_relaxed": 0.2722222222222222, + "em_relaxed_stderr": 0.014845038794433947 + }, + "e_tests.prompt2_qa_sk:chemia|0": { + "qem": 0.018518518518518517, + "qem_stderr": 0.018518518518518517, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 3.292181069958847e-05, + "rouge1_stderr": 3.292181069958849e-05, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 3.292181069958847e-05, + "rougeL_stderr": 3.292181069958849e-05 + }, + "sklep.prompt2_nli_binary_sk:rte|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "multi_wiki_qa.prompt1_qa_sk|0": { + "qem": 0.00040024014408645187, + "qem_stderr": 0.00028298419465801113, + "qf1": 0.004608242571473309, + "qf1_stderr": 0.00046536745363106924 + }, + "slovak_tests.prompt1_mqa_single_correct_sk:vyssie_studium|0": { + "acc": 0.21904761904761905, + "acc_stderr": 0.040556911537178254, + "em_relaxed": 0.21904761904761905, + "em_relaxed_stderr": 0.040556911537178254 + }, + "mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0": { + "acc": 0.17, + "acc_stderr": 0.03775251680686371, + "em_relaxed": 0.17, + "em_relaxed_stderr": 0.03775251680686371 + }, + "skj1.prompt3_skj1_sk|0": { + "acc": 0.75, + "acc_stderr": 0.04351941398892446, + "em_relaxed": 0.75, + "em_relaxed_stderr": 0.04351941398892446 + }, + "slovak_tests.prompt2_qa_sk:prijmacie_skusky|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0021574973031283713, + "rouge1_stderr": 0.0021574973031283713, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0010787486515641857, + "rougeL_stderr": 0.0010787486515641857 + }, + "e_tests.prompt1_mqa_multiple_correct_sk:geografia|0": { + "acc": 0.034482758620689655, + "acc_stderr": 0.034482758620689676 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0": { + "acc": 0.22777777777777777, + "acc_stderr": 0.03134731210385172, + "em_relaxed": 0.23333333333333334, + "em_relaxed_stderr": 0.031612960436120094 + }, + "scio.prompt2_mqa_single_correct_sk:mat|0": { + "acc": 0.21311475409836064, + "acc_stderr": 0.021434631795998253, + "em_relaxed": 0.22131147540983606, + "em_relaxed_stderr": 0.021728884019137824 + }, + "mmlu.prompt3_mqa_single_correct_sk:college_medicine|0": { + "acc": 0.20930232558139536, + "acc_stderr": 0.03110958390976464, + "em_relaxed": 0.21511627906976744, + "em_relaxed_stderr": 0.03142253684735939 + }, + "sklep.prompt3_hate_speech_sk:hate-speech|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0": { + "acc": 0.18666666666666668, + "acc_stderr": 0.03192085664169663, + "em_relaxed": 0.19333333333333333, + "em_relaxed_stderr": 0.03235245865324252 + }, + "scio.prompt2_mqa_single_correct_sk:che|0": { + "acc": 0.18699186991869918, + "acc_stderr": 0.024910116828719235, + "em_relaxed": 0.1910569105691057, + "em_relaxed_stderr": 0.025116395774587044 + }, + "skj1.prompt2_skj1_sk|0": { + "acc": 0.63, + "acc_stderr": 0.04852365870939099, + "em_relaxed": 0.64, + "em_relaxed_stderr": 0.048241815132442176 + }, + "truthfulqa.prompt3_mqa_single_correct_sk:mc1|0": { + "acc": 0.10282776349614396, + "acc_stderr": 0.010896388982494143, + "em_relaxed": 0.12853470437017994, + "em_relaxed_stderr": 0.012006729887990094 + }, + "e_tests.prompt3_qa_sk:informatika|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.006923076923076923, + "qf1_stderr": 0.006424094288133824, + "rouge1": 0.00046479944765467505, + "rouge1_stderr": 0.0001876188412456943, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.00046479944765467505, + "rougeL_stderr": 0.0001876188412456943 + }, + "mmlu.prompt1_mqa_single_correct_sk:astronomy|0": { + "acc": 0.23026315789473684, + "acc_stderr": 0.03426059424403165, + "em_relaxed": 0.23026315789473684, + "em_relaxed_stderr": 0.03426059424403165 + }, + "mmlu.prompt1_mqa_single_correct_sk:computer_security|0": { + "acc": 0.27, + "acc_stderr": 0.04461960433384741, + "em_relaxed": 0.28, + "em_relaxed_stderr": 0.04512608598542127 + }, + "mmlu.prompt2_mqa_single_correct_sk:nutrition|0": { + "acc": 0.2145214521452145, + "acc_stderr": 0.023621042274383643, + "em_relaxed": 0.24092409240924093, + "em_relaxed_stderr": 0.0246081660183802 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0": { + "acc": 0.2676767676767677, + "acc_stderr": 0.03154449888270286, + "em_relaxed": 0.2676767676767677, + "em_relaxed_stderr": 0.03154449888270286 + }, + "financial_exams.prompt3_completion_sk|0": { + "acc": 0.13343328335832083, + "acc_stderr": 0.009313609263721821, + "em_relaxed": 0.13343328335832083, + "em_relaxed_stderr": 0.009313609263721821 + }, + "mmlu.prompt3_mqa_single_correct_sk:sociology|0": { + "acc": 0.19289340101522842, + "acc_stderr": 0.028183576131702342, + "em_relaxed": 0.19289340101522842, + "em_relaxed_stderr": 0.028183576131702342 + }, + "mmlu.prompt2_mqa_single_correct_sk:world_religions|0": { + "acc": 0.26900584795321636, + "acc_stderr": 0.0340105262010409, + "em_relaxed": 0.2807017543859649, + "em_relaxed_stderr": 0.034462962170884265 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0": { + "acc": 0.15555555555555556, + "acc_stderr": 0.03130948364878314, + "em_relaxed": 0.15555555555555556, + "em_relaxed_stderr": 0.03130948364878314 + }, + "mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0": { + "acc": 0.18, + "acc_stderr": 0.03861229196653695, + "em_relaxed": 0.18, + "em_relaxed_stderr": 0.03861229196653695 + }, + "scio.prompt3_mqa_single_correct_sk:aj|0": { + "acc": 0.5, + "acc_stderr": 0.16666666666666666, + "em_relaxed": 0.5, + "em_relaxed_stderr": 0.16666666666666666 + }, + "e_tests.prompt3_mqa_single_correct_sk:geografia|0": { + "acc": 0.22448979591836735, + "acc_stderr": 0.06022425581505364, + "em_relaxed": 0.22448979591836735, + "em_relaxed_stderr": 0.06022425581505364 + }, + "mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0": { + "acc": 0.18181818181818182, + "acc_stderr": 0.03896103896103899, + "em_relaxed": 0.18181818181818182, + "em_relaxed_stderr": 0.03896103896103899 + }, + "e_tests.prompt1_mqa_multiple_correct_sk:fyzika|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "mmlu.prompt2_mqa_single_correct_sk:sociology|0": { + "acc": 0.15228426395939088, + "acc_stderr": 0.02566401503870399, + "em_relaxed": 0.15228426395939088, + "em_relaxed_stderr": 0.02566401503870399 + }, + "e_tests.prompt1_qa_sk:obcianska-nauka|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "e_tests.prompt3_mqa_single_correct_sk:matematika|0": { + "acc": 0.05128205128205128, + "acc_stderr": 0.035781575124326184, + "em_relaxed": 0.23076923076923078, + "em_relaxed_stderr": 0.0683479474337414 + }, + "mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0": { + "acc": 0.26515151515151514, + "acc_stderr": 0.0272187441820065, + "em_relaxed": 0.2765151515151515, + "em_relaxed_stderr": 0.027580130232496353 + }, + "e_tests.prompt3_mqa_multiple_correct_sk:dejepis|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0": { + "acc": 0.2, + "acc_stderr": 0.04020151261036846, + "em_relaxed": 0.22, + "em_relaxed_stderr": 0.04163331998932268 + }, + "mmlu.prompt2_mqa_single_correct_sk:management|0": { + "acc": 0.19, + "acc_stderr": 0.03942772444036622, + "em_relaxed": 0.19, + "em_relaxed_stderr": 0.03942772444036622 + }, + "mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0": { + "acc": 0.2206896551724138, + "acc_stderr": 0.03455930201924812, + "em_relaxed": 0.2413793103448276, + "em_relaxed_stderr": 0.03565998174135302 + }, + "sklep.prompt1_nli_3_classes_sk:nli|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "mmlu.prompt1_mqa_single_correct_sk:security_studies|0": { + "acc": 0.18468468468468469, + "acc_stderr": 0.026102500004017645, + "em_relaxed": 0.1981981981981982, + "em_relaxed_stderr": 0.026815582053277203 + }, + "sklep.prompt3_nli_binary_sk:rte|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0": { + "acc": 0.18624161073825504, + "acc_stderr": 0.015959806298600995, + "em_relaxed": 0.1912751677852349, + "em_relaxed_stderr": 0.01612394106868719 + }, + "mmlu.prompt1_mqa_single_correct_sk:nutrition|0": { + "acc": 0.22772277227722773, + "acc_stderr": 0.024131614233378947, + "em_relaxed": 0.264026402640264, + "em_relaxed_stderr": 0.0253659612714007 + }, + "mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0": { + "acc": 0.19, + "acc_stderr": 0.039427724440366234, + "em_relaxed": 0.19, + "em_relaxed_stderr": 0.039427724440366234 + }, + "mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0": { + "acc": 0.21, + "acc_stderr": 0.04093601807403326, + "em_relaxed": 0.22, + "em_relaxed_stderr": 0.04163331998932268 + }, + "mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0": { + "acc": 0.24646983311938384, + "acc_stderr": 0.015450495390660532, + "em_relaxed": 0.24775353016688062, + "em_relaxed_stderr": 0.015477478391132707 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0": { + "acc": 0.31666666666666665, + "acc_stderr": 0.034768900963930385, + "em_relaxed": 0.31666666666666665, + "em_relaxed_stderr": 0.034768900963930385 + }, + "arc.prompt2_mqa_single_correct_sk|0": { + "acc": 0.23353293413173654, + "acc_stderr": 0.012379394294094546, + "em_relaxed": 0.23353293413173654, + "em_relaxed_stderr": 0.012379394294094546 + }, + "slovak_tests.prompt2_mqa_single_correct_sk:monitor9|0": { + "acc": 0.14746543778801843, + "acc_stderr": 0.024125384514879104, + "em_relaxed": 0.14746543778801843, + "em_relaxed_stderr": 0.024125384514879104 + }, + "mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0": { + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "em_relaxed": 0.22, + "em_relaxed_stderr": 0.04163331998932269 + }, + "slovak_tests.prompt2_mqa_single_correct_sk:vyssie_studium|0": { + "acc": 0.11428571428571428, + "acc_stderr": 0.031197955091381097, + "em_relaxed": 0.11428571428571428, + "em_relaxed_stderr": 0.031197955091381097 + }, + "mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0": { + "acc": 0.2359249329758713, + "acc_stderr": 0.02201321444385349, + "em_relaxed": 0.2359249329758713, + "em_relaxed_stderr": 0.02201321444385349 + }, + "e_tests.prompt2_mqa_multiple_correct_sk:obcianska-nauka|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "e_tests.prompt2_mqa_multiple_correct_sk:biologia|0": { + "acc": 0.017857142857142856, + "acc_stderr": 0.017857142857142867 + }, + "toxic_sk.prompt2_toxicity_sk|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "e_tests.prompt2_mqa_single_correct_sk:fyzika|0": { + "acc": 0.13636363636363635, + "acc_stderr": 0.07488677009526491, + "em_relaxed": 0.22727272727272727, + "em_relaxed_stderr": 0.09144861547306321 + }, + "senti_sk.prompt1_sentiment_binary_sk|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0": { + "acc": 0.25288831835686776, + "acc_stderr": 0.015583584105316876, + "em_relaxed": 0.25417201540436457, + "em_relaxed_stderr": 0.01560965861721762 + }, + "slovak_tests.prompt2_qa_sk:monitor9|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0": { + "acc": 0.23049645390070922, + "acc_stderr": 0.025123739226872395, + "em_relaxed": 0.23404255319148937, + "em_relaxed_stderr": 0.025257861359432386 + }, + "e_tests.prompt1_mqa_single_correct_sk:informatika|0": { + "acc": 0.264, + "acc_stderr": 0.03958494337416042, + "em_relaxed": 0.272, + "em_relaxed_stderr": 0.0399612715739433 + }, + "e_tests.prompt3_mqa_multiple_correct_sk:slovensky-jazyk|0": { + "acc": 0.02142857142857143, + "acc_stderr": 0.012282463299882668 + }, + "e_tests.prompt3_mqa_multiple_correct_sk:geografia|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0": { + "acc": 0.2349799732977303, + "acc_stderr": 0.015502474593163047, + "em_relaxed": 0.2349799732977303, + "em_relaxed_stderr": 0.015502474593163047 + }, + "e_tests.prompt1_qa_sk:chemia|0": { + "qem": 0.018518518518518517, + "qem_stderr": 0.018518518518518517, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.00026455026455026457, + "rouge1_stderr": 0.00026455026455026484, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.00026455026455026457, + "rougeL_stderr": 0.00026455026455026484 + }, + "scio.prompt2_mqa_single_correct_sk:aj|0": { + "acc": 0.6, + "acc_stderr": 0.16329931618554522, + "em_relaxed": 0.6, + "em_relaxed_stderr": 0.16329931618554522 + }, + "e_tests.prompt2_mqa_multiple_correct_sk:informatika|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0": { + "acc": 0.19638242894056848, + "acc_stderr": 0.020220056896337255, + "em_relaxed": 0.20155038759689922, + "em_relaxed_stderr": 0.020418410166350903 + }, + "slovak_tests.prompt2_mqa_single_correct_sk:maturita|0": { + "acc": 0.15176151761517614, + "acc_stderr": 0.018703199518943774, + "em_relaxed": 0.15447154471544716, + "em_relaxed_stderr": 0.01883928643967625 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0": { + "acc": 0.37373737373737376, + "acc_stderr": 0.04887069039502487, + "em_relaxed": 0.3838383838383838, + "em_relaxed_stderr": 0.04912566964083466 + }, + "e_tests.prompt2_qa_sk:biologia|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0": { + "acc": 0.18, + "acc_stderr": 0.03861229196653697, + "em_relaxed": 0.18, + "em_relaxed_stderr": 0.03861229196653697 + }, + "e_tests.prompt1_qa_sk:informatika|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.00824175824175824, + "qf1_stderr": 0.006643857376866051, + "rouge1": 0.00033664858861620243, + "rouge1_stderr": 0.00019799367371799002, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.00033664858861620243, + "rougeL_stderr": 0.00019799367371799002 + }, + "mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0": { + "acc": 0.22784810126582278, + "acc_stderr": 0.03347528264060925, + "em_relaxed": 0.23417721518987342, + "em_relaxed_stderr": 0.03379766010403987 + }, + "sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "e_tests.prompt2_mqa_single_correct_sk:informatika|0": { + "acc": 0.224, + "acc_stderr": 0.037440705810827146, + "em_relaxed": 0.232, + "em_relaxed_stderr": 0.03790650637827826 + }, + "e_tests.prompt2_qa_sk:obcianska-nauka|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "mmlu.prompt1_mqa_single_correct_sk:management|0": { + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "em_relaxed": 0.21, + "em_relaxed_stderr": 0.040936018074033256 + }, + "mmlu.prompt2_mqa_single_correct_sk:international_law|0": { + "acc": 0.23728813559322035, + "acc_stderr": 0.03933012549934385, + "em_relaxed": 0.23728813559322035, + "em_relaxed_stderr": 0.03933012549934385 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0": { + "acc": 0.2488479262672811, + "acc_stderr": 0.029417385564063876, + "em_relaxed": 0.2626728110599078, + "em_relaxed_stderr": 0.029944068647646538 + }, + "e_tests.prompt3_qa_sk:fyzika|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0014626760356334343, + "rouge1_stderr": 0.0010249453223271077, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0014626760356334343, + "rougeL_stderr": 0.0010249453223271077 + }, + "mmlu.prompt3_mqa_single_correct_sk:marketing|0": { + "acc": 0.13247863247863248, + "acc_stderr": 0.022209309073165606, + "em_relaxed": 0.13247863247863248, + "em_relaxed_stderr": 0.022209309073165606 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0": { + "acc": 0.19172932330827067, + "acc_stderr": 0.024182419147367247, + "em_relaxed": 0.19924812030075187, + "em_relaxed_stderr": 0.02453709604824174 + }, + "slovak_tests.prompt1_qa_sk:maturita|0": { + "qem": 0.01327433628318584, + "qem_stderr": 0.007629803632081149, + "qf1": 0.00454104547024901, + "qf1_stderr": 0.0026612932202592188, + "rouge1": 0.001362195307144739, + "rouge1_stderr": 0.0004933551702641079, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0012709627553702657, + "rougeL_stderr": 0.0004501834684090662 + }, + "e_tests.prompt1_mqa_multiple_correct_sk:dejepis|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "senti_sk.prompt3_sentiment_3_class_sk|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0": { + "acc": 0.1919191919191919, + "acc_stderr": 0.028057791672989017, + "em_relaxed": 0.1919191919191919, + "em_relaxed_stderr": 0.028057791672989017 + }, + "e_tests.prompt1_qa_sk:dejepis|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "e_tests.prompt1_mqa_multiple_correct_sk:biologia|0": { + "acc": 0.017857142857142856, + "acc_stderr": 0.017857142857142867 + }, + "senti_sk.prompt1_sentiment_3_class_sk|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "mmlu.prompt3_mqa_single_correct_sk:college_biology|0": { + "acc": 0.2323943661971831, + "acc_stderr": 0.03556905428023101, + "em_relaxed": 0.2323943661971831, + "em_relaxed_stderr": 0.03556905428023101 + }, + "slohry.prompt2_identity_form_sk:identity_form_mid|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt1_declension_sk:declension_mid|0": { + "acc": 0.011976047904191617, + "acc_stderr": 0.005960989845848317, + "em_norm_relaxed": 0.011976047904191617, + "em_norm_relaxed_stderr": 0.005960989845848317 + }, + "slohry.prompt1_declension_sk:declension_low|0": { + "acc": 0.0029940119760479044, + "acc_stderr": 0.0029940119760478844, + "em_norm_relaxed": 0.0029940119760479044, + "em_norm_relaxed_stderr": 0.0029940119760478844 + }, + "analogies_sk.prompt1_analogy_sk|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "slovak_trivia.prompt3_mqa_single_correct_sk|0": { + "acc": 0.22096317280453256, + "acc_stderr": 0.022114015575065166, + "em_relaxed": 0.2237960339943343, + "em_relaxed_stderr": 0.02221481955471835 + }, + "slohry.prompt3_identity_form_sk:identity_form_low|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt3_gradation_sk:gradation_high|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt1_inflection_sk:inflection_high|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt1_gradation_sk:gradation_low|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "chiby.prompt2_chiby_sk|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt3_inflection_sk:inflection_high|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slovak_trivia.prompt2_mqa_single_correct_sk|0": { + "acc": 0.1501416430594901, + "acc_stderr": 0.01903936786327736, + "em_relaxed": 0.1558073654390935, + "em_relaxed_stderr": 0.019330515671326005 + }, + "chiby.prompt3_chiby_sk|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt1_gradation_sk:gradation_mid|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt1_declension_sk:declension_high|0": { + "acc": 0.017964071856287425, + "acc_stderr": 0.0072785348116838205, + "em_norm_relaxed": 0.017964071856287425, + "em_norm_relaxed_stderr": 0.0072785348116838205 + }, + "slohry.prompt2_inflection_sk:inflection_low|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt2_declension_sk:declension_low|0": { + "acc": 0.0029940119760479044, + "acc_stderr": 0.0029940119760478896, + "em_norm_relaxed": 0.0029940119760479044, + "em_norm_relaxed_stderr": 0.0029940119760478896 + }, + "slohry.prompt3_gradation_sk:gradation_low|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt2_gradation_sk:gradation_mid|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "chiby.prompt1_chiby_sk|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt3_declension_sk:declension_low|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt3_identity_form_sk:identity_form_high|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt1_identity_form_sk:identity_form_mid|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt3_declension_sk:declension_high|0": { + "acc": 0.011976047904191617, + "acc_stderr": 0.005960989845848319, + "em_norm_relaxed": 0.011976047904191617, + "em_norm_relaxed_stderr": 0.005960989845848319 + }, + "slohry.prompt3_declension_sk:declension_mid|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt1_inflection_sk:inflection_low|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slovak_trivia.prompt1_mqa_single_correct_sk|0": { + "acc": 0.22662889518413598, + "acc_stderr": 0.02231414649207437, + "em_relaxed": 0.23229461756373937, + "em_relaxed_stderr": 0.022508447165267422 + }, + "slohry.prompt1_identity_form_sk:identity_form_low|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt2_declension_sk:declension_high|0": { + "acc": 0.020958083832335328, + "acc_stderr": 0.007849722658097238, + "em_norm_relaxed": 0.020958083832335328, + "em_norm_relaxed_stderr": 0.007849722658097238 + }, + "slohry.prompt2_inflection_sk:inflection_high|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt2_declension_sk:declension_mid|0": { + "acc": 0.014970059880239521, + "acc_stderr": 0.006654483730236385, + "em_norm_relaxed": 0.014970059880239521, + "em_norm_relaxed_stderr": 0.006654483730236385 + }, + "analogies_sk.prompt2_analogy_sk|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "slohry.prompt1_gradation_sk:gradation_high|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt3_gradation_sk:gradation_mid|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt3_inflection_sk:inflection_mid|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt3_identity_form_sk:identity_form_mid|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt2_gradation_sk:gradation_low|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "analogies_sk.prompt3_analogy_sk|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "slohry.prompt1_identity_form_sk:identity_form_high|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt2_identity_form_sk:identity_form_low|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt2_inflection_sk:inflection_mid|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt3_inflection_sk:inflection_low|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt2_gradation_sk:gradation_high|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt1_inflection_sk:inflection_mid|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt2_identity_form_sk:identity_form_high|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "e_tests_mqa_SK_AVERAGE|0": { + "em_relaxed": 0.21755858691850297, + "acc": 0.15008726003490402, + "em_relaxed_stderr": 0.03505820050399378, + "acc_stderr": 0.027287029108301525, + "em_relaxed__prompts": { + "1": 0.23819517313746066, + "2": 0.2088142707240294, + "3": 0.20566631689401887 + }, + "acc__prompts": { + "1": 0.16454749439042632, + "2": 0.14360508601346297, + "3": 0.14210919970082272 + } + }, + "e_tests_mqa_AVERAGE|0": { + "em_relaxed": 0.21755858691850297, + "em_relaxed_stderr": 0.03505820050399378, + "acc": 0.15008726003490402, + "acc_stderr": 0.027287029108301525 + }, + "mmlu_SK_AVERAGE|0": { + "em_relaxed": 0.22816822334507222, + "acc": 0.2224518960853366, + "em_relaxed_stderr": 0.025975113405158696, + "acc_stderr": 0.02571374453987217, + "em_relaxed__prompts": { + "1": 0.24628693921298422, + "2": 0.21719491655182974, + "3": 0.22102281427040268 + }, + "acc__prompts": { + "1": 0.2399326290001531, + "2": 0.21137651201959884, + "3": 0.21604654723625785 + } + }, + "mmlu_AVERAGE|0": { + "em_relaxed": 0.22816822334507222, + "em_relaxed_stderr": 0.025975113405158696, + "acc": 0.2224518960853366, + "acc_stderr": 0.02571374453987217 + }, + "scio_SK_AVERAGE|0": { + "em_relaxed": 0.24911297852474323, + "acc": 0.238468720821662, + "em_relaxed_stderr": 0.020694508860887174, + "acc_stderr": 0.0204222231602884, + "em_relaxed__prompts": { + "1": 0.261624649859944, + "3": 0.23865546218487396, + "2": 0.24705882352941178 + }, + "acc__prompts": { + "1": 0.2504201680672269, + "3": 0.2280112044817927, + "2": 0.23697478991596638 + } + }, + "scio_AVERAGE|0": { + "em_relaxed": 0.24911297852474323, + "em_relaxed_stderr": 0.020694508860887174, + "acc": 0.238468720821662, + "acc_stderr": 0.0204222231602884 + }, + "slovak_tests_qa_SK_AVERAGE|0": { + "rouge2": 0.0, + "rouge1": 0.0011809129165421361, + "rougeL": 0.0011384377140699951, + "qf1": 0.003009101538513303, + "qem": 0.012605042016806723, + "rouge2_stderr": 0.0, + "rouge1_stderr": 0.0007262749376038493, + "rougeL_stderr": 0.0006990122730613884, + "qf1_stderr": 0.0019761043274910683, + "qem_stderr": 0.0072451076506316785, + "rouge2__prompts": { + "3": 0.0, + "2": 0.0, + "1": 0.0 + }, + "rouge1__prompts": { + "3": 0.001309304483429979, + "2": 0.0008896015463691078, + "1": 0.001343832719827322 + }, + "rougeL__prompts": { + "3": 0.001309304483429979, + "2": 0.0008488085301334873, + "1": 0.0012572001286465196 + }, + "qf1__prompts": { + "3": 0.0021008403361344537, + "2": 0.0026143790849673205, + "1": 0.004312085194438136 + }, + "qem__prompts": { + "3": 0.012605042016806723, + "2": 0.012605042016806723, + "1": 0.012605042016806723 + } + }, + "slovak_tests_qa_AVERAGE|0": { + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rouge1": 0.0011809129165421361, + "rouge1_stderr": 0.0007262749376038493, + "rougeL": 0.0011384377140699951, + "rougeL_stderr": 0.0006990122730613884, + "qf1": 0.003009101538513303, + "qf1_stderr": 0.0019761043274910683, + "qem": 0.012605042016806723, + "qem_stderr": 0.0072451076506316785 + }, + "slovak_tests_mqa_SK_AVERAGE|0": { + "em_relaxed": 0.17543859649122806, + "acc": 0.17389966143428748, + "em_relaxed_stderr": 0.02232972578404266, + "acc_stderr": 0.02225738292492629, + "em_relaxed__prompts": { + "1": 0.20498614958448755, + "3": 0.17820867959372114, + "2": 0.14312096029547552 + }, + "acc__prompts": { + "1": 0.20221606648199447, + "3": 0.17820867959372114, + "2": 0.14127423822714683 + } + }, + "slovak_tests_mqa_AVERAGE|0": { + "em_relaxed": 0.17543859649122806, + "em_relaxed_stderr": 0.02232972578404266, + "acc": 0.17389966143428748, + "acc_stderr": 0.02225738292492629 + }, + "movie_review_SK_AVERAGE|0": { + "em_norm_relaxed": 0.0, + "acc": 0.0, + "em_norm_relaxed_stderr": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed__prompts": { + "2": 0.0, + "3": 0.0, + "1": 0.0 + }, + "acc__prompts": { + "2": 0.0, + "3": 0.0, + "1": 0.0 + } + }, + "movie_review_AVERAGE|0": { + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0, + "acc": 0.0, + "acc_stderr": 0.0 + }, + "e_tests_qa_SK_AVERAGE|0": { + "rouge2": 0.0, + "rouge1": 0.00029664081444656757, + "rougeL": 0.00029664081444656757, + "qf1": 0.0026099107814683436, + "qem": 0.002257336343115124, + "rouge2_stderr": 0.0, + "rouge1_stderr": 0.0001935712320593315, + "rougeL_stderr": 0.0001935712320593315, + "qf1_stderr": 0.002326519969477271, + "qem_stderr": 0.002257336343115124, + "rouge2__prompts": { + "1": 0.0, + "2": 0.0, + "3": 0.0 + }, + "rouge1__prompts": { + "1": 0.00011697710542177868, + "2": 0.0002733713466381956, + "3": 0.0004995739912797284 + }, + "rougeL__prompts": { + "1": 0.00011697710542177868, + "2": 0.0002733713466381956, + "3": 0.0004995739912797284 + }, + "qf1__prompts": { + "1": 0.0025798129635601417, + "2": 0.00290228958400516, + "3": 0.002347629796839729 + }, + "qem__prompts": { + "1": 0.002257336343115124, + "2": 0.002257336343115124, + "3": 0.002257336343115124 + } + }, + "e_tests_qa_AVERAGE|0": { + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rouge1": 0.00029664081444656757, + "rouge1_stderr": 0.0001935712320593315, + "rougeL": 0.00029664081444656757, + "rougeL_stderr": 0.0001935712320593315, + "qf1": 0.0026099107814683436, + "qf1_stderr": 0.002326519969477271, + "qem": 0.002257336343115124, + "qem_stderr": 0.002257336343115124 + }, + "sklep_nli_SK_AVERAGE|0": { + "em_norm_relaxed": 0.0, + "acc": 0.0, + "em_norm_relaxed_stderr": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed__prompts": { + "3": 0.0, + "2": 0.0, + "1": 0.0 + }, + "acc__prompts": { + "3": 0.0, + "2": 0.0, + "1": 0.0 + } + }, + "sklep_nli_AVERAGE|0": { + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0, + "acc": 0.0, + "acc_stderr": 0.0 + }, + "sklep_sentiment_analysis_SK_AVERAGE|0": { + "em_norm_relaxed": 0.0, + "acc": 0.0, + "em_norm_relaxed_stderr": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed__prompts": { + "2": 0.0, + "1": 0.0, + "3": 0.0 + }, + "acc__prompts": { + "2": 0.0, + "1": 0.0, + "3": 0.0 + } + }, + "sklep_sentiment_analysis_AVERAGE|0": { + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0, + "acc": 0.0, + "acc_stderr": 0.0 + }, + "arc_SK_AVERAGE|0": { + "em_relaxed": 0.24493869404049046, + "acc": 0.24493869404049046, + "em_relaxed_stderr": 0.012565550837641016, + "acc_stderr": 0.012565550837641016, + "em_relaxed__prompts": { + "1": 0.2728828058169376, + "3": 0.22840034217279725, + "2": 0.23353293413173654 + }, + "acc__prompts": { + "1": 0.2728828058169376, + "3": 0.22840034217279725, + "2": 0.23353293413173654 + } + }, + "arc_AVERAGE|0": { + "em_relaxed": 0.24493869404049046, + "em_relaxed_stderr": 0.012565550837641016, + "acc": 0.24493869404049046, + "acc_stderr": 0.012565550837641016 + }, + "belebele_SK_AVERAGE|0": { + "em_relaxed": 0.2611111111111111, + "acc": 0.2537037037037037, + "em_relaxed_stderr": 0.014646428295956382, + "acc_stderr": 0.014509896842640889, + "em_relaxed__prompts": { + "2": 0.25666666666666665, + "3": 0.2544444444444444, + "1": 0.2722222222222222 + }, + "acc__prompts": { + "2": 0.2511111111111111, + "3": 0.24666666666666667, + "1": 0.2633333333333333 + } + }, + "belebele_AVERAGE|0": { + "em_relaxed": 0.2611111111111111, + "em_relaxed_stderr": 0.014646428295956382, + "acc": 0.2537037037037037, + "acc_stderr": 0.014509896842640889 + }, + "senti_sk_3class_SK_AVERAGE|0": { + "em_norm_relaxed": 0.0, + "acc": 0.0, + "em_norm_relaxed_stderr": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed__prompts": { + "2": 0.0, + "3": 0.0, + "1": 0.0 + }, + "acc__prompts": { + "2": 0.0, + "3": 0.0, + "1": 0.0 + } + }, + "senti_sk_3class_AVERAGE|0": { + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0, + "acc": 0.0, + "acc_stderr": 0.0 + }, + "sklep_hate_speech_SK_AVERAGE|0": { + "em_norm_relaxed": 0.0, + "acc": 0.0, + "em_norm_relaxed_stderr": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed__prompts": { + "2": 0.0, + "1": 0.0, + "3": 0.0 + }, + "acc__prompts": { + "2": 0.0, + "1": 0.0, + "3": 0.0 + } + }, + "sklep_hate_speech_AVERAGE|0": { + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0, + "acc": 0.0, + "acc_stderr": 0.0 + }, + "slovakcopa_SK_AVERAGE|0": { + "em_relaxed": 0.05333333333333334, + "acc": 0.05333333333333334, + "em_relaxed_stderr": 0.007876087666852084, + "acc_stderr": 0.007876087666852084, + "em_relaxed__prompts": { + "3": 0.144, + "2": 0.006, + "1": 0.01 + }, + "acc__prompts": { + "3": 0.144, + "2": 0.006, + "1": 0.01 + } + }, + "slovakcopa_AVERAGE|0": { + "em_relaxed": 0.05333333333333334, + "em_relaxed_stderr": 0.007876087666852084, + "acc": 0.05333333333333334, + "acc_stderr": 0.007876087666852084 + }, + "globalmmlu_lite_SK_AVERAGE|0": { + "em_relaxed": 0.21333333333333335, + "acc": 0.21083333333333334, + "em_relaxed_stderr": 0.020468272436489743, + "acc_stderr": 0.020379605193633, + "em_relaxed__prompts": { + "1": 0.2425, + "3": 0.205, + "2": 0.1925 + }, + "acc__prompts": { + "1": 0.24, + "3": 0.2025, + "2": 0.19 + } + }, + "globalmmlu_lite_AVERAGE|0": { + "em_relaxed": 0.21333333333333335, + "em_relaxed_stderr": 0.020468272436489743, + "acc": 0.21083333333333334, + "acc_stderr": 0.020379605193633 + }, + "sklep_qa_SK_AVERAGE|0": { + "qf1": 0.0070376260418807705, + "qem": 0.00027827054854081884, + "qf1_stderr": 0.0003824771262616922, + "qem_stderr": 0.0001679351079586058, + "qf1__prompts": { + "3": 0.006093522810123945, + "2": 0.006743145308527306, + "1": 0.008276210006991061 + }, + "qem__prompts": { + "3": 0.0002087029114056141, + "2": 0.0002087029114056141, + "1": 0.0004174058228112282 + } + }, + "sklep_qa_AVERAGE|0": { + "qf1": 0.0070376260418807705, + "qf1_stderr": 0.0003824771262616922, + "qem": 0.00027827054854081884, + "qem_stderr": 0.0001679351079586058 + }, + "financial_exams_SK_AVERAGE|0": { + "em_relaxed": 0.08995502248875563, + "acc": 0.0887056471764118, + "em_relaxed_stderr": 0.007689948855002123, + "acc_stderr": 0.007626104510560348, + "em_relaxed__prompts": { + "1": 0.05697151424287856, + "2": 0.07946026986506746, + "3": 0.13343328335832083 + }, + "acc__prompts": { + "1": 0.053973013493253376, + "2": 0.07871064467766117, + "3": 0.13343328335832083 + } + }, + "financial_exams_AVERAGE|0": { + "em_relaxed": 0.08995502248875563, + "em_relaxed_stderr": 0.007689948855002123, + "acc": 0.0887056471764118, + "acc_stderr": 0.007626104510560348 + }, + "ifeval_sk_SK_AVERAGE|0": { + "prompt_level_strict_acc": 0.0782122905027933, + "prompt_level_loose_acc": 0.0782122905027933, + "inst_level_strict_acc": 0.17312348668280872, + "inst_level_loose_acc": 0.17312348668280872, + "prompt_level_strict_acc_stderr": 0.011597665111537078, + "prompt_level_loose_acc_stderr": 0.011597665111537078, + "inst_level_strict_acc_stderr": 0.0004210039776560394, + "inst_level_loose_acc_stderr": 0.0004210039776560394 + }, + "ifeval_sk_AVERAGE|0": { + "prompt_level_strict_acc": 0.0782122905027933, + "prompt_level_strict_acc_stderr": 0.011597665111537078, + "prompt_level_loose_acc": 0.0782122905027933, + "prompt_level_loose_acc_stderr": 0.011597665111537078, + "inst_level_strict_acc": 0.17312348668280872, + "inst_level_strict_acc_stderr": 0.0004210039776560394, + "inst_level_loose_acc": 0.17312348668280872, + "inst_level_loose_acc_stderr": 0.0004210039776560394 + }, + "toxic_sk_SK_AVERAGE|0": { + "em_norm_relaxed": 0.0, + "acc": 0.0, + "em_norm_relaxed_stderr": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed__prompts": { + "1": 0.0, + "3": 0.0, + "2": 0.0 + }, + "acc__prompts": { + "1": 0.0, + "3": 0.0, + "2": 0.0 + } + }, + "toxic_sk_AVERAGE|0": { + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0, + "acc": 0.0, + "acc_stderr": 0.0 + }, + "driving_test_SK_AVERAGE|0": { + "em_relaxed": 0.1407513076557299, + "acc": 0.13742272943414172, + "em_relaxed_stderr": 0.013052324431028241, + "acc_stderr": 0.012921192285229888, + "em_relaxed__prompts": { + "2": 0.10271041369472182, + "1": 0.16833095577746077, + "3": 0.15121255349500715 + }, + "acc__prompts": { + "2": 0.09985734664764621, + "1": 0.16405135520684735, + "3": 0.14835948644793154 + } + }, + "driving_test_AVERAGE|0": { + "em_relaxed": 0.1407513076557299, + "em_relaxed_stderr": 0.013052324431028241, + "acc": 0.13742272943414172, + "acc_stderr": 0.012921192285229888 + }, + "sklep_rte_SK_AVERAGE|0": { + "em_norm_relaxed": 0.0, + "acc": 0.0, + "em_norm_relaxed_stderr": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed__prompts": { + "1": 0.0, + "2": 0.0, + "3": 0.0 + }, + "acc__prompts": { + "1": 0.0, + "2": 0.0, + "3": 0.0 + } + }, + "sklep_rte_AVERAGE|0": { + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0, + "acc": 0.0, + "acc_stderr": 0.0 + }, + "multi_wiki_qa_SK_AVERAGE|0": { + "qf1": 0.005263913929238519, + "qem": 0.0005336535254486025, + "qf1_stderr": 0.0005228307644937593, + "qem_stderr": 0.00032536056243172176, + "qf1__prompts": { + "3": 0.005765729879562343, + "2": 0.005417769336679904, + "1": 0.004608242571473309 + }, + "qem__prompts": { + "3": 0.0006003602161296778, + "2": 0.0006003602161296778, + "1": 0.00040024014408645187 + } + }, + "multi_wiki_qa_AVERAGE|0": { + "qf1": 0.005263913929238519, + "qf1_stderr": 0.0005228307644937593, + "qem": 0.0005336535254486025, + "qem_stderr": 0.00032536056243172176 + }, + "truthful_qa_SK_AVERAGE|0": { + "em_relaxed": 0.09682947729220223, + "acc": 0.06469580119965725, + "em_relaxed_stderr": 0.010525690171591523, + "acc_stderr": 0.008619652748939895, + "em_relaxed__prompts": { + "2": 0.08354755784061697, + "1": 0.07840616966580977, + "3": 0.12853470437017994 + }, + "acc__prompts": { + "2": 0.04884318766066838, + "1": 0.042416452442159386, + "3": 0.10282776349614396 + } + }, + "truthful_qa_AVERAGE|0": { + "em_relaxed": 0.09682947729220223, + "em_relaxed_stderr": 0.010525690171591523, + "acc": 0.06469580119965725, + "acc_stderr": 0.008619652748939895 + }, + "senti_sk_binary_SK_AVERAGE|0": { + "em_norm_relaxed": 0.0, + "acc": 0.0, + "em_norm_relaxed_stderr": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed__prompts": { + "3": 0.0, + "2": 0.0, + "1": 0.0 + }, + "acc__prompts": { + "3": 0.0, + "2": 0.0, + "1": 0.0 + } + }, + "senti_sk_binary_AVERAGE|0": { + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0, + "acc": 0.0, + "acc_stderr": 0.0 + }, + "skj1_SK_AVERAGE|0": { + "em_relaxed": 0.61, + "acc": 0.5966666666666667, + "em_relaxed_stderr": 0.04721666475945084, + "acc_stderr": 0.04715805991356215, + "em_relaxed__prompts": { + "1": 0.44, + "3": 0.75, + "2": 0.64 + }, + "acc__prompts": { + "1": 0.41, + "3": 0.75, + "2": 0.63 + } + }, + "skj1_AVERAGE|0": { + "em_relaxed": 0.61, + "em_relaxed_stderr": 0.04721666475945084, + "acc": 0.5966666666666667, + "acc_stderr": 0.04715805991356215 + }, + "slohry_SK_AVERAGE|0": { + "em_norm_relaxed": 0.002331002331002331, + "acc": 0.002331002331002331, + "em_norm_relaxed_stderr": 0.0011036777204322752, + "acc_stderr": 0.0011036777204322752, + "em_norm_relaxed__prompts": { + "2": 0.003246753246753247, + "1": 0.0027472527472527475, + "3": 0.000999000999000999 + }, + "acc__prompts": { + "2": 0.003246753246753247, + "1": 0.0027472527472527475, + "3": 0.000999000999000999 + } + }, + "slohry_AVERAGE|0": { + "em_norm_relaxed": 0.002331002331002331, + "em_norm_relaxed_stderr": 0.0011036777204322752, + "acc": 0.002331002331002331, + "acc_stderr": 0.0011036777204322752 + }, + "analogies_sk_SK_AVERAGE|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "acc__prompts": { + "1": 0.0, + "2": 0.0, + "3": 0.0 + } + }, + "analogies_sk_AVERAGE|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "slovak_trivia_SK_AVERAGE|0": { + "em_relaxed": 0.20396600566572237, + "acc": 0.19924457034938622, + "em_relaxed_stderr": 0.021351260797103925, + "acc_stderr": 0.021155843310138964, + "em_relaxed__prompts": { + "3": 0.2237960339943343, + "2": 0.1558073654390935, + "1": 0.23229461756373937 + }, + "acc__prompts": { + "3": 0.22096317280453256, + "2": 0.1501416430594901, + "1": 0.22662889518413598 + } + }, + "slovak_trivia_AVERAGE|0": { + "em_relaxed": 0.20396600566572237, + "em_relaxed_stderr": 0.021351260797103925, + "acc": 0.19924457034938622, + "acc_stderr": 0.021155843310138964 + }, + "chiby_SK_AVERAGE|0": { + "em_norm_relaxed": 0.0, + "acc": 0.0, + "em_norm_relaxed_stderr": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed__prompts": { + "2": 0.0, + "3": 0.0, + "1": 0.0 + }, + "acc__prompts": { + "2": 0.0, + "3": 0.0, + "1": 0.0 + } + }, + "chiby_AVERAGE|0": { + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0, + "acc": 0.0, + "acc_stderr": 0.0 + }, + "SK_AVERAGE|0": { + "em_relaxed": 0.21419205155386342, + "acc": 0.10986595916434654, + "rouge2": 0.0, + "rouge1": 0.0007387768654943519, + "rougeL": 0.0007175392642582814, + "qf1": 0.004480138072775234, + "qem": 0.003918575608477817, + "em_norm_relaxed": 0.0002331002331002331, + "prompt_level_strict_acc": 0.0782122905027933, + "prompt_level_loose_acc": 0.0782122905027933, + "inst_level_strict_acc": 0.17312348668280872, + "inst_level_loose_acc": 0.17312348668280872, + "em_relaxed_stderr": 0.006300951163940428, + "acc_stderr": 0.003263889036835735, + "rouge2_stderr": 0.0, + "rouge1_stderr": 0.00037581415183320424, + "rougeL_stderr": 0.0003626596130572774, + "qf1_stderr": 0.0007801171483309406, + "qem_stderr": 0.0018993619641089511, + "em_norm_relaxed_stderr": 0.00011036777204322752, + "prompt_level_strict_acc_stderr": 0.011597665111537078, + "prompt_level_loose_acc_stderr": 0.011597665111537078, + "inst_level_strict_acc_stderr": 0.0004210039776560394, + "inst_level_loose_acc_stderr": 0.0004210039776560394 + }, + "AVERAGE|0": { + "em_relaxed": 0.18120805585001873, + "acc": 0.09273256151262048, + "rouge2": 0.0, + "rouge1": 0.0007387768654943519, + "rougeL": 0.0007175392642582814, + "qf1": 0.004480138072775234, + "qem": 0.003918575608477817, + "em_norm_relaxed": 0.000259000259000259, + "prompt_level_strict_acc": 0.0782122905027933, + "prompt_level_loose_acc": 0.0782122905027933, + "inst_level_strict_acc": 0.17312348668280872, + "inst_level_loose_acc": 0.17312348668280872, + "em_relaxed_stderr": 0.005577871787987613, + "acc_stderr": 0.002843082460505095, + "rouge2_stderr": 0.0, + "rouge1_stderr": 0.00037581415183320424, + "rougeL_stderr": 0.0003626596130572774, + "qf1_stderr": 0.0007801171483309406, + "qem_stderr": 0.0018993619641089511, + "em_norm_relaxed_stderr": 0.00012263085782580836, + "prompt_level_strict_acc_stderr": 0.011597665111537078, + "prompt_level_loose_acc_stderr": 0.011597665111537078, + "inst_level_strict_acc_stderr": 0.0004210039776560394, + "inst_level_loose_acc_stderr": 0.0004210039776560394 + } + } +} \ No newline at end of file diff --git a/perun/ministral_3b_aya/sk_bench/aggregated/models__ministral-last-109064.json b/perun/ministral_3b_aya/sk_bench/aggregated/models__ministral-last-109064.json new file mode 100644 index 0000000..7b35c52 --- /dev/null +++ b/perun/ministral_3b_aya/sk_bench/aggregated/models__ministral-last-109064.json @@ -0,0 +1,3250 @@ +{ + "model_name": "models/ministral-last-109064", + "config_general": { + "lighteval_sha": "?", + "num_fewshot_seeds": 1, + "max_samples": null, + "job_id": "0", + "start_time": 883355.295419078, + "end_time": 883503.394399547, + "total_evaluation_time_secondes": "148.0989804690471", + "model_config": { + "model_name": "models/ministral-last-109064", + "generation_parameters": { + "num_blocks": null, + "block_size": null, + "early_stopping": null, + "repetition_penalty": null, + "frequency_penalty": null, + "length_penalty": null, + "presence_penalty": null, + "max_new_tokens": null, + "min_new_tokens": null, + "seed": null, + "stop_tokens": null, + "temperature": 0, + "top_k": null, + "min_p": null, + "top_p": null, + "truncate_prompt": null, + "cache_implementation": null, + "response_format": null, + "skip_special_tokens": true + }, + "system_prompt": null, + "cache_dir": "~/.cache/huggingface/lighteval", + "tokenizer": null, + "revision": "main", + "dtype": "bfloat16", + "tensor_parallel_size": 1, + "data_parallel_size": 1, + "pipeline_parallel_size": 1, + "gpu_memory_utilization": 0.9, + "enable_prefix_caching": null, + "max_model_length": null, + "quantization": null, + "load_format": null, + "swap_space": 4, + "seed": 1234, + "trust_remote_code": true, + "add_special_tokens": true, + "multichoice_continuations_start_space": true, + "pairwise_tokenization": false, + "max_num_seqs": 128, + "max_num_batched_tokens": 2048, + "subfolder": null, + "is_async": false, + "override_chat_template": null, + "enforce_eager": false + }, + "model_name": "models/ministral-last-109064" + }, + "versions": {}, + "results": { + "movie_review.prompt1_sentiment_binary_sk|0": { + "acc": 0.09860557768924302, + "acc_stderr": 0.006654792030522998, + "em_norm_relaxed": 0.10109561752988047, + "em_norm_relaxed_stderr": 0.006728979843112432 + }, + "e_tests.prompt2_mqa_multiple_correct_sk:fyzika|0": { + "acc": 0.1, + "acc_stderr": 0.06882472016116853 + }, + "mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0": { + "acc": 0.21983914209115282, + "acc_stderr": 0.021472030675572816, + "em_relaxed": 0.21983914209115282, + "em_relaxed_stderr": 0.021472030675572816 + }, + "mmlu.prompt3_mqa_single_correct_sk:marketing|0": { + "acc": 0.27350427350427353, + "acc_stderr": 0.029202540153431183, + "em_relaxed": 0.27350427350427353, + "em_relaxed_stderr": 0.029202540153431183 + }, + "belebele.prompt3_mqa_single_correct_sk|0": { + "acc": 0.27111111111111114, + "acc_stderr": 0.014826016446581967, + "em_relaxed": 0.2722222222222222, + "em_relaxed_stderr": 0.014845038794433947 + }, + "e_tests.prompt1_mqa_single_correct_sk:obcianska-nauka|0": { + "acc": 0.2318840579710145, + "acc_stderr": 0.05117930441535768, + "em_relaxed": 0.2318840579710145, + "em_relaxed_stderr": 0.05117930441535768 + }, + "mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0": { + "acc": 0.254601226993865, + "acc_stderr": 0.024164783161712056, + "em_relaxed": 0.254601226993865, + "em_relaxed_stderr": 0.024164783161712056 + }, + "mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0": { + "acc": 0.20202020202020202, + "acc_stderr": 0.0405583530232473, + "em_relaxed": 0.2222222222222222, + "em_relaxed_stderr": 0.041996052556580815 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0": { + "acc": 0.20522388059701493, + "acc_stderr": 0.017460611985170203, + "em_relaxed": 0.20522388059701493, + "em_relaxed_stderr": 0.017460611985170203 + }, + "senti_sk.prompt2_sentiment_binary_sk|0": { + "acc": 0.32135193133047213, + "acc_stderr": 0.007649500362664599, + "em_norm_relaxed": 0.3261802575107296, + "em_norm_relaxed_stderr": 0.007679288875174346 + }, + "slovak_tests.prompt2_mqa_single_correct_sk:vyssie_studium|0": { + "acc": 0.18095238095238095, + "acc_stderr": 0.03775026958386237, + "em_relaxed": 0.18095238095238095, + "em_relaxed_stderr": 0.03775026958386237 + }, + "e_tests.prompt1_mqa_multiple_correct_sk:biologia|0": { + "acc": 0.05357142857142857, + "acc_stderr": 0.030361917118846844 + }, + "mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0": { + "acc": 0.27181208053691275, + "acc_stderr": 0.018238857466939602, + "em_relaxed": 0.27181208053691275, + "em_relaxed_stderr": 0.018238857466939602 + }, + "e_tests.prompt3_qa_sk:chemia|0": { + "qem": 0.037037037037037035, + "qem_stderr": 0.025940896147128693, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.02444444444444445, + "rouge1_stderr": 0.011223747742795042, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.02444444444444445, + "rougeL_stderr": 0.011223747742795042 + }, + "scio.prompt3_mqa_single_correct_sk:vsp|0": { + "acc": 0.2657766990291262, + "acc_stderr": 0.015398294286338463, + "em_relaxed": 0.27184466019417475, + "em_relaxed_stderr": 0.015508596797488939 + }, + "arc.prompt2_mqa_single_correct_sk|0": { + "acc": 0.24123182207014543, + "acc_stderr": 0.01251844632394384, + "em_relaxed": 0.24721984602224123, + "em_relaxed_stderr": 0.012622759999132595 + }, + "sklep.prompt3_nli_binary_sk:rte|0": { + "acc": 0.1650602409638554, + "acc_stderr": 0.009114348186638069, + "em_norm_relaxed": 0.16566265060240964, + "em_norm_relaxed_stderr": 0.009127670467992892 + }, + "mmlu.prompt1_mqa_single_correct_sk:astronomy|0": { + "acc": 0.1513157894736842, + "acc_stderr": 0.029162631596843982, + "em_relaxed": 0.1513157894736842, + "em_relaxed_stderr": 0.029162631596843982 + }, + "mmlu.prompt3_mqa_single_correct_sk:machine_learning|0": { + "acc": 0.25892857142857145, + "acc_stderr": 0.041577515398656284, + "em_relaxed": 0.2767857142857143, + "em_relaxed_stderr": 0.04246624336697624 + }, + "driving_test.prompt3_mqa_single_correct_sk|0": { + "acc": 0.17261055634807418, + "acc_stderr": 0.014283664605079543, + "em_relaxed": 0.17261055634807418, + "em_relaxed_stderr": 0.014283664605079543 + }, + "e_tests.prompt3_mqa_multiple_correct_sk:dejepis|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0": { + "acc": 0.23232323232323232, + "acc_stderr": 0.04266016017054689, + "em_relaxed": 0.23232323232323232, + "em_relaxed_stderr": 0.04266016017054689 + }, + "mmlu.prompt2_mqa_single_correct_sk:professional_law|0": { + "acc": 0.24679760888129804, + "acc_stderr": 0.01260472537669998, + "em_relaxed": 0.2493595217762596, + "em_relaxed_stderr": 0.012648412983648835 + }, + "mmlu.prompt3_mqa_single_correct_sk:formal_logic|0": { + "acc": 0.25217391304347825, + "acc_stderr": 0.040672227541547167, + "em_relaxed": 0.2608695652173913, + "em_relaxed_stderr": 0.04112631751856164 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0": { + "acc": 0.2676767676767677, + "acc_stderr": 0.03154449888270286, + "em_relaxed": 0.2676767676767677, + "em_relaxed_stderr": 0.03154449888270286 + }, + "senti_sk.prompt1_sentiment_binary_sk|0": { + "acc": 0.39914163090128757, + "acc_stderr": 0.0080217638856134, + "em_norm_relaxed": 0.4069206008583691, + "em_norm_relaxed_stderr": 0.008046954795106477 + }, + "sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0": { + "acc": 0.1833013435700576, + "acc_stderr": 0.011991910538467281, + "em_norm_relaxed": 0.19193857965451055, + "em_norm_relaxed_stderr": 0.012206129123492155 + }, + "mmlu.prompt3_mqa_single_correct_sk:world_religions|0": { + "acc": 0.2222222222222222, + "acc_stderr": 0.03188578017686398, + "em_relaxed": 0.2222222222222222, + "em_relaxed_stderr": 0.03188578017686398 + }, + "e_tests.prompt2_qa_sk:fyzika|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.014285714285714287, + "rouge1_stderr": 0.014285714285714287, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.014285714285714287, + "rougeL_stderr": 0.014285714285714287 + }, + "scio.prompt2_mqa_single_correct_sk:che|0": { + "acc": 0.2601626016260163, + "acc_stderr": 0.028028995361669362, + "em_relaxed": 0.2601626016260163, + "em_relaxed_stderr": 0.028028995361669362 + }, + "mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0": { + "acc": 0.17, + "acc_stderr": 0.0377525168068637, + "em_relaxed": 0.17, + "em_relaxed_stderr": 0.0377525168068637 + }, + "mmlu.prompt1_mqa_single_correct_sk:professional_law|0": { + "acc": 0.2408198121263877, + "acc_stderr": 0.012500448888138267, + "em_relaxed": 0.2408198121263877, + "em_relaxed_stderr": 0.012500448888138267 + }, + "mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0": { + "acc": 0.25153374233128833, + "acc_stderr": 0.024068141649530538, + "em_relaxed": 0.25153374233128833, + "em_relaxed_stderr": 0.024068141649530538 + }, + "mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0": { + "acc": 0.22826086956521738, + "acc_stderr": 0.025309563050037266, + "em_relaxed": 0.22826086956521738, + "em_relaxed_stderr": 0.025309563050037266 + }, + "e_tests.prompt2_qa_sk:geografia|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.01904761904761905, + "qf1_stderr": 0.019047619047619056, + "rouge1": 0.01360544217687075, + "rouge1_stderr": 0.013605442176870748, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.01360544217687075, + "rougeL_stderr": 0.013605442176870748 + }, + "mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0": { + "acc": 0.29292929292929293, + "acc_stderr": 0.045972676254181784, + "em_relaxed": 0.29292929292929293, + "em_relaxed_stderr": 0.045972676254181784 + }, + "e_tests.prompt1_mqa_multiple_correct_sk:fyzika|0": { + "acc": 0.05, + "acc_stderr": 0.04999999999999998 + }, + "e_tests.prompt2_mqa_single_correct_sk:fyzika|0": { + "acc": 0.13636363636363635, + "acc_stderr": 0.07488677009526491, + "em_relaxed": 0.13636363636363635, + "em_relaxed_stderr": 0.07488677009526491 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0": { + "acc": 0.24067164179104478, + "acc_stderr": 0.018482060138670608, + "em_relaxed": 0.24253731343283583, + "em_relaxed_stderr": 0.01853075065058218 + }, + "slovakcopa.prompt1_copa_sk|0": { + "acc": 0.402, + "acc_stderr": 0.021948929609938612, + "em_relaxed": 0.402, + "em_relaxed_stderr": 0.021948929609938612 + }, + "movie_review.prompt3_sentiment_binary_sk|0": { + "acc": 0.20766932270916336, + "acc_stderr": 0.009054526051113992, + "em_norm_relaxed": 0.20916334661354583, + "em_norm_relaxed_stderr": 0.009078466587423147 + }, + "mmlu.prompt2_mqa_single_correct_sk:college_medicine|0": { + "acc": 0.20348837209302326, + "acc_stderr": 0.03078703062133896, + "em_relaxed": 0.20348837209302326, + "em_relaxed_stderr": 0.03078703062133896 + }, + "e_tests.prompt3_mqa_multiple_correct_sk:geografia|0": { + "acc": 0.034482758620689655, + "acc_stderr": 0.034482758620689676 + }, + "e_tests.prompt3_mqa_single_correct_sk:geografia|0": { + "acc": 0.20408163265306123, + "acc_stderr": 0.05817221556628253, + "em_relaxed": 0.20408163265306123, + "em_relaxed_stderr": 0.05817221556628253 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0": { + "acc": 0.22932330827067668, + "acc_stderr": 0.02582481829225878, + "em_relaxed": 0.22932330827067668, + "em_relaxed_stderr": 0.02582481829225878 + }, + "sklep.prompt2_qa_sk:question-answering|0": { + "qem": 0.05238443076280914, + "qem_stderr": 0.0022760883977866997, + "qf1": 0.14428713100967, + "qf1_stderr": 0.002729924096699559 + }, + "e_tests.prompt2_qa_sk:informatika|0": { + "qem": 0.038461538461538464, + "qem_stderr": 0.021915495473007307, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0036324786324786326, + "rouge1_stderr": 0.0025563439430967027, + "rouge2": 0.0008547008547008547, + "rouge2_stderr": 0.0008547008547008546, + "rougeL": 0.0036324786324786326, + "rougeL_stderr": 0.0025563439430967027 + }, + "mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0": { + "acc": 0.32340425531914896, + "acc_stderr": 0.030579442773610337, + "em_relaxed": 0.32340425531914896, + "em_relaxed_stderr": 0.030579442773610337 + }, + "e_tests.prompt1_qa_sk:dejepis|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0": { + "acc": 0.2482758620689655, + "acc_stderr": 0.03600105692727771, + "em_relaxed": 0.2482758620689655, + "em_relaxed_stderr": 0.03600105692727771 + }, + "mmlu.prompt3_mqa_single_correct_sk:human_aging|0": { + "acc": 0.3926940639269406, + "acc_stderr": 0.03307521564995785, + "em_relaxed": 0.3926940639269406, + "em_relaxed_stderr": 0.03307521564995785 + }, + "mmlu.prompt3_mqa_single_correct_sk:global_facts|0": { + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "em_relaxed": 0.31, + "em_relaxed_stderr": 0.04648231987117316 + }, + "e_tests.prompt1_qa_sk:obcianska-nauka|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.020408163265306124, + "rouge1_stderr": 0.020408163265306124, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.020408163265306124, + "rougeL_stderr": 0.020408163265306124 + }, + "mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0": { + "acc": 0.2548262548262548, + "acc_stderr": 0.027129450992473495, + "em_relaxed": 0.2625482625482625, + "em_relaxed_stderr": 0.027394382974098772 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0": { + "acc": 0.19626168224299065, + "acc_stderr": 0.027213555922450262, + "em_relaxed": 0.205607476635514, + "em_relaxed_stderr": 0.027691547344010747 + }, + "mmlu.prompt1_mqa_single_correct_sk:formal_logic|0": { + "acc": 0.23478260869565218, + "acc_stderr": 0.039698395317531235, + "em_relaxed": 0.23478260869565218, + "em_relaxed_stderr": 0.039698395317531235 + }, + "senti_sk.prompt3_sentiment_3_class_sk|0": { + "acc": 0.573444515715202, + "acc_stderr": 0.007232629453324713, + "em_norm_relaxed": 0.5809279452640581, + "em_norm_relaxed_stderr": 0.007215529950821413 + }, + "e_tests.prompt1_qa_sk:biologia|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.019047619047619046, + "qf1_stderr": 0.019047619047619035, + "rouge1": 0.04, + "rouge1_stderr": 0.022529159443043678, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.04, + "rougeL_stderr": 0.022529159443043678 + }, + "slovak_tests.prompt3_mqa_single_correct_sk:maturita|0": { + "acc": 0.27371273712737126, + "acc_stderr": 0.023242230741504617, + "em_relaxed": 0.27371273712737126, + "em_relaxed_stderr": 0.023242230741504617 + }, + "mmlu.prompt1_mqa_single_correct_sk:virology|0": { + "acc": 0.35365853658536583, + "acc_stderr": 0.03744805613781578, + "em_relaxed": 0.35365853658536583, + "em_relaxed_stderr": 0.03744805613781578 + }, + "mmlu.prompt1_mqa_single_correct_sk:sociology|0": { + "acc": 0.23857868020304568, + "acc_stderr": 0.03044390069569734, + "em_relaxed": 0.23857868020304568, + "em_relaxed_stderr": 0.03044390069569734 + }, + "mmlu.prompt2_mqa_single_correct_sk:human_aging|0": { + "acc": 0.3926940639269406, + "acc_stderr": 0.03307521564995785, + "em_relaxed": 0.3926940639269406, + "em_relaxed_stderr": 0.03307521564995785 + }, + "e_tests.prompt2_mqa_multiple_correct_sk:chemia|0": { + "acc": 0.022727272727272728, + "acc_stderr": 0.022727272727272728 + }, + "mmlu.prompt2_mqa_single_correct_sk:virology|0": { + "acc": 0.2926829268292683, + "acc_stderr": 0.035637888362588285, + "em_relaxed": 0.2926829268292683, + "em_relaxed_stderr": 0.035637888362588285 + }, + "mmlu.prompt3_mqa_single_correct_sk:college_biology|0": { + "acc": 0.2323943661971831, + "acc_stderr": 0.03556905428023102, + "em_relaxed": 0.23943661971830985, + "em_relaxed_stderr": 0.03593796120702438 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0": { + "acc": 0.2534562211981567, + "acc_stderr": 0.02959731008909016, + "em_relaxed": 0.2534562211981567, + "em_relaxed_stderr": 0.02959731008909016 + }, + "multi_wiki_qa.prompt3_qa_sk|0": { + "qem": 0.07884730838503101, + "qem_stderr": 0.0038128351969214953, + "qf1": 0.12171966052606027, + "qf1_stderr": 0.003955037338259647 + }, + "e_tests.prompt3_mqa_multiple_correct_sk:slovensky-jazyk|0": { + "acc": 0.02857142857142857, + "acc_stderr": 0.014130710990583526 + }, + "e_tests.prompt2_mqa_single_correct_sk:biologia|0": { + "acc": 0.3103448275862069, + "acc_stderr": 0.049887188500387446, + "em_relaxed": 0.3103448275862069, + "em_relaxed_stderr": 0.049887188500387446 + }, + "mmlu.prompt3_mqa_single_correct_sk:business_ethics|0": { + "acc": 0.24, + "acc_stderr": 0.04292346959909284, + "em_relaxed": 0.24, + "em_relaxed_stderr": 0.04292346959909284 + }, + "e_tests.prompt1_mqa_multiple_correct_sk:geografia|0": { + "acc": 0.034482758620689655, + "acc_stderr": 0.034482758620689676 + }, + "mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0": { + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "em_relaxed": 0.29, + "em_relaxed_stderr": 0.045604802157206845 + }, + "ifeval_sk.ifeval|0": { + "inst_level_loose_acc": 0.21307506053268765, + "inst_level_loose_acc_stderr": 0.00041218013349389904, + "inst_level_strict_acc": 0.21307506053268765, + "inst_level_strict_acc_stderr": 0.00041218013349389904, + "prompt_level_loose_acc": 0.10242085661080075, + "prompt_level_loose_acc_stderr": 0.01309628698247887, + "prompt_level_strict_acc": 0.10242085661080075, + "prompt_level_strict_acc_stderr": 0.01309628698247887 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0": { + "acc": 0.18181818181818182, + "acc_stderr": 0.027479603010538804, + "em_relaxed": 0.18181818181818182, + "em_relaxed_stderr": 0.027479603010538804 + }, + "e_tests.prompt1_qa_sk:geografia|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0": { + "acc": 0.22429906542056074, + "acc_stderr": 0.015251413306651283, + "em_relaxed": 0.22429906542056074, + "em_relaxed_stderr": 0.015251413306651283 + }, + "e_tests.prompt3_qa_sk:slovensky-jazyk|0": { + "qem": 0.0051813471502590676, + "qem_stderr": 0.00518134715025908, + "qf1": 0.008737173625927054, + "qf1_stderr": 0.005740091786106941, + "rouge1": 0.016785287251608494, + "rouge1_stderr": 0.006781696658405232, + "rouge2": 0.00659444182760245, + "rouge2_stderr": 0.005363480846922639, + "rougeL": 0.016785287251608494, + "rougeL_stderr": 0.006781696658405232 + }, + "mmlu.prompt1_mqa_single_correct_sk:marketing|0": { + "acc": 0.26495726495726496, + "acc_stderr": 0.028911208802749472, + "em_relaxed": 0.26495726495726496, + "em_relaxed_stderr": 0.028911208802749472 + }, + "driving_test.prompt1_mqa_single_correct_sk|0": { + "acc": 0.3666191155492154, + "acc_stderr": 0.01821339929295096, + "em_relaxed": 0.3666191155492154, + "em_relaxed_stderr": 0.01821339929295096 + }, + "scio.prompt1_mqa_single_correct_sk:aj|0": { + "acc": 0.2, + "acc_stderr": 0.13333333333333333, + "em_relaxed": 0.2, + "em_relaxed_stderr": 0.13333333333333333 + }, + "sklep.prompt2_nli_binary_sk:rte|0": { + "acc": 0.5132530120481927, + "acc_stderr": 0.012271402375351256, + "em_norm_relaxed": 0.5132530120481927, + "em_norm_relaxed_stderr": 0.012271402375351256 + }, + "mmlu.prompt3_mqa_single_correct_sk:sociology|0": { + "acc": 0.23857868020304568, + "acc_stderr": 0.03044390069569734, + "em_relaxed": 0.23857868020304568, + "em_relaxed_stderr": 0.03044390069569734 + }, + "multi_wiki_qa.prompt2_qa_sk|0": { + "qem": 0.08204922953772263, + "qem_stderr": 0.003882716972284686, + "qf1": 0.12697709233547141, + "qf1_stderr": 0.0040052643032774445 + }, + "e_tests.prompt2_mqa_single_correct_sk:geografia|0": { + "acc": 0.22448979591836735, + "acc_stderr": 0.06022425581505364, + "em_relaxed": 0.22448979591836735, + "em_relaxed_stderr": 0.06022425581505364 + }, + "mmlu.prompt3_mqa_single_correct_sk:computer_security|0": { + "acc": 0.26, + "acc_stderr": 0.0440844002276808, + "em_relaxed": 0.26, + "em_relaxed_stderr": 0.0440844002276808 + }, + "mmlu.prompt3_mqa_single_correct_sk:nutrition|0": { + "acc": 0.22442244224422442, + "acc_stderr": 0.024007242951044422, + "em_relaxed": 0.2376237623762376, + "em_relaxed_stderr": 0.0244921064409582 + }, + "mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0": { + "acc": 0.2, + "acc_stderr": 0.04020151261036845, + "em_relaxed": 0.2, + "em_relaxed_stderr": 0.04020151261036845 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0": { + "acc": 0.21705426356589147, + "acc_stderr": 0.020982456910832083, + "em_relaxed": 0.21705426356589147, + "em_relaxed_stderr": 0.020982456910832083 + }, + "globalmmlu-lite.prompt3_mqa_single_correct_sk|0": { + "acc": 0.2875, + "acc_stderr": 0.022658174179374095, + "em_relaxed": 0.2925, + "em_relaxed_stderr": 0.022774020616430368 + }, + "mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0": { + "acc": 0.23166023166023167, + "acc_stderr": 0.02626591571601146, + "em_relaxed": 0.23166023166023167, + "em_relaxed_stderr": 0.02626591571601146 + }, + "e_tests.prompt2_mqa_multiple_correct_sk:geografia|0": { + "acc": 0.034482758620689655, + "acc_stderr": 0.034482758620689676 + }, + "mmlu.prompt1_mqa_single_correct_sk:prehistory|0": { + "acc": 0.2413793103448276, + "acc_stderr": 0.02399654973117778, + "em_relaxed": 0.2413793103448276, + "em_relaxed_stderr": 0.02399654973117778 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0": { + "acc": 0.18666666666666668, + "acc_stderr": 0.0319208566416966, + "em_relaxed": 0.18666666666666668, + "em_relaxed_stderr": 0.0319208566416966 + }, + "sklep.prompt3_nli_3_classes_sk:nli|0": { + "acc": 0.002398081534772182, + "acc_stderr": 0.0006915050545011868, + "em_norm_relaxed": 0.002398081534772182, + "em_norm_relaxed_stderr": 0.0006915050545011868 + }, + "e_tests.prompt3_qa_sk:fyzika|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.014285714285714287, + "rouge1_stderr": 0.014285714285714287, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.014285714285714287, + "rougeL_stderr": 0.014285714285714287 + }, + "mmlu.prompt1_mqa_single_correct_sk:econometrics|0": { + "acc": 0.25892857142857145, + "acc_stderr": 0.041577515398656284, + "em_relaxed": 0.25892857142857145, + "em_relaxed_stderr": 0.041577515398656284 + }, + "mmlu.prompt3_mqa_single_correct_sk:virology|0": { + "acc": 0.3048780487804878, + "acc_stderr": 0.03605784583600454, + "em_relaxed": 0.3048780487804878, + "em_relaxed_stderr": 0.03605784583600454 + }, + "e_tests.prompt3_mqa_single_correct_sk:informatika|0": { + "acc": 0.224, + "acc_stderr": 0.03744070581082714, + "em_relaxed": 0.224, + "em_relaxed_stderr": 0.03744070581082714 + }, + "mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0": { + "acc": 0.28498074454428757, + "acc_stderr": 0.016183663085909714, + "em_relaxed": 0.28498074454428757, + "em_relaxed_stderr": 0.016183663085909714 + }, + "mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0": { + "acc": 0.25838926174496646, + "acc_stderr": 0.01794596235855172, + "em_relaxed": 0.26174496644295303, + "em_relaxed_stderr": 0.01802120781376133 + }, + "arc.prompt3_mqa_single_correct_sk|0": { + "acc": 0.24550898203592814, + "acc_stderr": 0.012593293013246194, + "em_relaxed": 0.24550898203592814, + "em_relaxed_stderr": 0.012593293013246194 + }, + "mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0": { + "acc": 0.3191489361702128, + "acc_stderr": 0.030472973363380035, + "em_relaxed": 0.3191489361702128, + "em_relaxed_stderr": 0.030472973363380035 + }, + "scio.prompt1_mqa_single_correct_sk:vsp|0": { + "acc": 0.24514563106796117, + "acc_stderr": 0.014994906005686719, + "em_relaxed": 0.24514563106796117, + "em_relaxed_stderr": 0.014994906005686719 + }, + "e_tests.prompt2_mqa_multiple_correct_sk:biologia|0": { + "acc": 0.03571428571428571, + "acc_stderr": 0.025023180348048964 + }, + "e_tests.prompt1_mqa_single_correct_sk:slovensky-jazyk|0": { + "acc": 0.27074235807860264, + "acc_stderr": 0.02078550104939707, + "em_relaxed": 0.27074235807860264, + "em_relaxed_stderr": 0.02078550104939707 + }, + "mmlu.prompt1_mqa_single_correct_sk:world_religions|0": { + "acc": 0.2631578947368421, + "acc_stderr": 0.03377310252209194, + "em_relaxed": 0.2631578947368421, + "em_relaxed_stderr": 0.03377310252209194 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0": { + "acc": 0.26262626262626265, + "acc_stderr": 0.04445287676983945, + "em_relaxed": 0.26262626262626265, + "em_relaxed_stderr": 0.04445287676983945 + }, + "mmlu.prompt2_mqa_single_correct_sk:econometrics|0": { + "acc": 0.26785714285714285, + "acc_stderr": 0.04203277291467762, + "em_relaxed": 0.2857142857142857, + "em_relaxed_stderr": 0.04287858751340456 + }, + "mmlu.prompt2_mqa_single_correct_sk:security_studies|0": { + "acc": 0.16216216216216217, + "acc_stderr": 0.02479468591119148, + "em_relaxed": 0.16666666666666666, + "em_relaxed_stderr": 0.02506903489984114 + }, + "belebele.prompt1_mqa_single_correct_sk|0": { + "acc": 0.2577777777777778, + "acc_stderr": 0.014588474089651667, + "em_relaxed": 0.2577777777777778, + "em_relaxed_stderr": 0.014588474089651667 + }, + "mmlu.prompt3_mqa_single_correct_sk:econometrics|0": { + "acc": 0.26785714285714285, + "acc_stderr": 0.04203277291467762, + "em_relaxed": 0.2767857142857143, + "em_relaxed_stderr": 0.042466243366976256 + }, + "slovak_tests.prompt1_qa_sk:prijmacie_skusky|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.024691358024691357, + "qf1_stderr": 0.024691358024691353, + "rouge1": 0.01851851851851852, + "rouge1_stderr": 0.01851851851851852, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.01851851851851852, + "rougeL_stderr": 0.01851851851851852 + }, + "movie_review.prompt2_sentiment_binary_sk|0": { + "acc": 0.13097609561752988, + "acc_stderr": 0.007530749388005803, + "em_norm_relaxed": 0.13645418326693226, + "em_norm_relaxed_stderr": 0.007662357895460915 + }, + "e_tests.prompt3_mqa_single_correct_sk:obcianska-nauka|0": { + "acc": 0.13043478260869565, + "acc_stderr": 0.04084071463598805, + "em_relaxed": 0.14492753623188406, + "em_relaxed_stderr": 0.042689636687312106 + }, + "e_tests.prompt2_mqa_multiple_correct_sk:matematika|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "mmlu.prompt3_mqa_single_correct_sk:college_medicine|0": { + "acc": 0.20930232558139536, + "acc_stderr": 0.03110958390976464, + "em_relaxed": 0.20930232558139536, + "em_relaxed_stderr": 0.03110958390976464 + }, + "mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0": { + "acc": 0.2277992277992278, + "acc_stderr": 0.026111473881583542, + "em_relaxed": 0.2277992277992278, + "em_relaxed_stderr": 0.026111473881583542 + }, + "mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0": { + "acc": 0.2734274711168164, + "acc_stderr": 0.015979778720725094, + "em_relaxed": 0.2734274711168164, + "em_relaxed_stderr": 0.015979778720725094 + }, + "mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0": { + "acc": 0.22784810126582278, + "acc_stderr": 0.033475282640609255, + "em_relaxed": 0.22784810126582278, + "em_relaxed_stderr": 0.033475282640609255 + }, + "mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0": { + "acc": 0.29906542056074764, + "acc_stderr": 0.044470182376718334, + "em_relaxed": 0.29906542056074764, + "em_relaxed_stderr": 0.044470182376718334 + }, + "mmlu.prompt2_mqa_single_correct_sk:college_physics|0": { + "acc": 0.21568627450980393, + "acc_stderr": 0.04092563958237656, + "em_relaxed": 0.21568627450980393, + "em_relaxed_stderr": 0.04092563958237656 + }, + "mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0": { + "acc": 0.2064343163538874, + "acc_stderr": 0.020985093876713833, + "em_relaxed": 0.2064343163538874, + "em_relaxed_stderr": 0.020985093876713833 + }, + "slovak_tests.prompt1_mqa_single_correct_sk:maturita|0": { + "acc": 0.25203252032520324, + "acc_stderr": 0.022633188466788714, + "em_relaxed": 0.25203252032520324, + "em_relaxed_stderr": 0.022633188466788714 + }, + "e_tests.prompt3_qa_sk:dejepis|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "scio.prompt1_mqa_single_correct_sk:mat|0": { + "acc": 0.1885245901639344, + "acc_stderr": 0.020472706124868213, + "em_relaxed": 0.1912568306010929, + "em_relaxed_stderr": 0.020585781457546286 + }, + "e_tests.prompt3_mqa_multiple_correct_sk:informatika|0": { + "acc": 0.02857142857142857, + "acc_stderr": 0.028571428571428574 + }, + "scio.prompt3_mqa_single_correct_sk:mat|0": { + "acc": 0.31420765027322406, + "acc_stderr": 0.024297313836707593, + "em_relaxed": 0.31693989071038253, + "em_relaxed_stderr": 0.024354066150157113 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0": { + "acc": 0.22340425531914893, + "acc_stderr": 0.024847921358063962, + "em_relaxed": 0.23404255319148937, + "em_relaxed_stderr": 0.025257861359432414 + }, + "slovak_tests.prompt3_mqa_single_correct_sk:vyssie_studium|0": { + "acc": 0.14285714285714285, + "acc_stderr": 0.03431317581537582, + "em_relaxed": 0.14285714285714285, + "em_relaxed_stderr": 0.03431317581537582 + }, + "slovakcopa.prompt2_copa_sk|0": { + "acc": 0.392, + "acc_stderr": 0.021854684955611256, + "em_relaxed": 0.394, + "em_relaxed_stderr": 0.021874299301689253 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0": { + "acc": 0.22962962962962963, + "acc_stderr": 0.036333844140734636, + "em_relaxed": 0.22962962962962963, + "em_relaxed_stderr": 0.036333844140734636 + }, + "e_tests.prompt2_mqa_multiple_correct_sk:slovensky-jazyk|0": { + "acc": 0.02857142857142857, + "acc_stderr": 0.014130710990583526 + }, + "belebele.prompt2_mqa_single_correct_sk|0": { + "acc": 0.27111111111111114, + "acc_stderr": 0.014826016446581962, + "em_relaxed": 0.27111111111111114, + "em_relaxed_stderr": 0.014826016446581962 + }, + "skj1.prompt1_skj1_sk|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_relaxed": 0.01, + "em_relaxed_stderr": 0.009999999999999997 + }, + "mmlu.prompt1_mqa_single_correct_sk:computer_security|0": { + "acc": 0.26, + "acc_stderr": 0.0440844002276808, + "em_relaxed": 0.26, + "em_relaxed_stderr": 0.0440844002276808 + }, + "driving_test.prompt2_mqa_single_correct_sk|0": { + "acc": 0.2738944365192582, + "acc_stderr": 0.01685553212123784, + "em_relaxed": 0.2738944365192582, + "em_relaxed_stderr": 0.01685553212123784 + }, + "scio.prompt3_mqa_single_correct_sk:aj|0": { + "acc": 0.3, + "acc_stderr": 0.15275252316519464, + "em_relaxed": 0.3, + "em_relaxed_stderr": 0.15275252316519464 + }, + "mmlu.prompt1_mqa_single_correct_sk:global_facts|0": { + "acc": 0.26, + "acc_stderr": 0.04408440022768078, + "em_relaxed": 0.26, + "em_relaxed_stderr": 0.04408440022768078 + }, + "scio.prompt1_mqa_single_correct_sk:bio|0": { + "acc": 0.21238938053097345, + "acc_stderr": 0.022246609446382862, + "em_relaxed": 0.21238938053097345, + "em_relaxed_stderr": 0.022246609446382862 + }, + "mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0": { + "acc": 0.24, + "acc_stderr": 0.04292346959909283, + "em_relaxed": 0.24, + "em_relaxed_stderr": 0.04292346959909283 + }, + "skj1.prompt3_skj1_sk|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_relaxed": 0.0, + "em_relaxed_stderr": 0.0 + }, + "skj1.prompt2_skj1_sk|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_relaxed": 0.0, + "em_relaxed_stderr": 0.0 + }, + "e_tests.prompt2_mqa_multiple_correct_sk:informatika|0": { + "acc": 0.02857142857142857, + "acc_stderr": 0.028571428571428574 + }, + "mmlu.prompt1_mqa_single_correct_sk:international_law|0": { + "acc": 0.22033898305084745, + "acc_stderr": 0.03831824849223319, + "em_relaxed": 0.22033898305084745, + "em_relaxed_stderr": 0.03831824849223319 + }, + "e_tests.prompt1_mqa_single_correct_sk:dejepis|0": { + "acc": 0.1875, + "acc_stderr": 0.05693291124629694, + "em_relaxed": 0.1875, + "em_relaxed_stderr": 0.05693291124629694 + }, + "mmlu.prompt2_mqa_single_correct_sk:world_religions|0": { + "acc": 0.23391812865497075, + "acc_stderr": 0.03246721765117826, + "em_relaxed": 0.23391812865497075, + "em_relaxed_stderr": 0.03246721765117826 + }, + "slovak_tests.prompt1_qa_sk:maturita|0": { + "qem": 0.008849557522123894, + "qem_stderr": 0.0062436609110668876, + "qf1": 0.007079646017699116, + "qf1_stderr": 0.0038256704785137783, + "rouge1": 0.014790713408591503, + "rouge1_stderr": 0.004864012429457399, + "rouge2": 0.003448518050287962, + "rouge2_stderr": 0.002428770392002376, + "rougeL": 0.014790713408591503, + "rougeL_stderr": 0.004864012429457399 + }, + "sklep.prompt1_qa_sk:question-answering|0": { + "qem": 0.05248878221851195, + "qem_stderr": 0.002278228840555749, + "qf1": 0.1424834912487429, + "qf1_stderr": 0.0027244148341408567 + }, + "globalmmlu-lite.prompt1_mqa_single_correct_sk|0": { + "acc": 0.2725, + "acc_stderr": 0.022290166077429768, + "em_relaxed": 0.2725, + "em_relaxed_stderr": 0.022290166077429768 + }, + "mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0": { + "acc": 0.24846625766871167, + "acc_stderr": 0.02396990269668754, + "em_relaxed": 0.24846625766871167, + "em_relaxed_stderr": 0.02396990269668754 + }, + "mmlu.prompt3_mqa_single_correct_sk:security_studies|0": { + "acc": 0.18468468468468469, + "acc_stderr": 0.026102500004017648, + "em_relaxed": 0.18468468468468469, + "em_relaxed_stderr": 0.026102500004017648 + }, + "e_tests.prompt2_mqa_multiple_correct_sk:obcianska-nauka|0": { + "acc": 0.03571428571428571, + "acc_stderr": 0.03571428571428572 + }, + "sklep.prompt1_nli_3_classes_sk:nli|0": { + "acc": 0.004396482813749001, + "acc_stderr": 0.0009353639970831211, + "em_norm_relaxed": 0.004996003197442046, + "em_norm_relaxed_stderr": 0.0009968011147772275 + }, + "e_tests.prompt2_qa_sk:obcianska-nauka|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.020408163265306124, + "rouge1_stderr": 0.020408163265306124, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.020408163265306124, + "rougeL_stderr": 0.020408163265306124 + }, + "mmlu.prompt2_mqa_single_correct_sk:marketing|0": { + "acc": 0.24358974358974358, + "acc_stderr": 0.028120966503914394, + "em_relaxed": 0.24358974358974358, + "em_relaxed_stderr": 0.028120966503914394 + }, + "financial_exams.prompt3_completion_sk|0": { + "acc": 0.18590704647676162, + "acc_stderr": 0.010655402235931219, + "em_relaxed": 0.18590704647676162, + "em_relaxed_stderr": 0.010655402235931219 + }, + "arc.prompt1_mqa_single_correct_sk|0": { + "acc": 0.24721984602224123, + "acc_stderr": 0.012622759999132588, + "em_relaxed": 0.24721984602224123, + "em_relaxed_stderr": 0.012622759999132588 + }, + "slovak_tests.prompt1_mqa_single_correct_sk:monitor9|0": { + "acc": 0.30414746543778803, + "acc_stderr": 0.0313021257639245, + "em_relaxed": 0.30414746543778803, + "em_relaxed_stderr": 0.0313021257639245 + }, + "sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0": { + "acc": 0.09213051823416507, + "acc_stderr": 0.00896371626415267, + "em_norm_relaxed": 0.09404990403071017, + "em_norm_relaxed_stderr": 0.009047028350861394 + }, + "e_tests.prompt2_mqa_single_correct_sk:matematika|0": { + "acc": 0.15384615384615385, + "acc_stderr": 0.05852968540405248, + "em_relaxed": 0.15384615384615385, + "em_relaxed_stderr": 0.05852968540405248 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0": { + "acc": 0.20567375886524822, + "acc_stderr": 0.024112138950471887, + "em_relaxed": 0.20567375886524822, + "em_relaxed_stderr": 0.024112138950471887 + }, + "mmlu.prompt2_mqa_single_correct_sk:prehistory|0": { + "acc": 0.2601880877742947, + "acc_stderr": 0.024603155110002158, + "em_relaxed": 0.26332288401253917, + "em_relaxed_stderr": 0.02469842907233297 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0": { + "acc": 0.1822429906542056, + "acc_stderr": 0.026451344562470145, + "em_relaxed": 0.1822429906542056, + "em_relaxed_stderr": 0.026451344562470145 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0": { + "acc": 0.22807017543859648, + "acc_stderr": 0.056069817847611766, + "em_relaxed": 0.22807017543859648, + "em_relaxed_stderr": 0.056069817847611766 + }, + "e_tests.prompt1_qa_sk:fyzika|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.014285714285714287, + "rouge1_stderr": 0.014285714285714287, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.014285714285714287, + "rougeL_stderr": 0.014285714285714287 + }, + "mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0": { + "acc": 0.24242424242424243, + "acc_stderr": 0.04329004329004326, + "em_relaxed": 0.24242424242424243, + "em_relaxed_stderr": 0.04329004329004326 + }, + "mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0": { + "acc": 0.19, + "acc_stderr": 0.03942772444036623, + "em_relaxed": 0.19, + "em_relaxed_stderr": 0.03942772444036623 + }, + "globalmmlu-lite.prompt2_mqa_single_correct_sk|0": { + "acc": 0.2675, + "acc_stderr": 0.022160484939004153, + "em_relaxed": 0.27, + "em_relaxed_stderr": 0.022225772481648733 + }, + "e_tests.prompt2_qa_sk:slovensky-jazyk|0": { + "qem": 0.0051813471502590676, + "qem_stderr": 0.00518134715025908, + "qf1": 0.010905502097211939, + "qf1_stderr": 0.0062654994299230354, + "rouge1": 0.015304902351534477, + "rouge1_stderr": 0.006747971135704163, + "rouge2": 0.0072538860103626935, + "rouge2_stderr": 0.005570450232424934, + "rougeL": 0.015304902351534477, + "rougeL_stderr": 0.006747971135704163 + }, + "slovak_tests.prompt3_mqa_single_correct_sk:monitor9|0": { + "acc": 0.30414746543778803, + "acc_stderr": 0.0313021257639245, + "em_relaxed": 0.30414746543778803, + "em_relaxed_stderr": 0.0313021257639245 + }, + "mmlu.prompt1_mqa_single_correct_sk:human_aging|0": { + "acc": 0.3515981735159817, + "acc_stderr": 0.03233829906905826, + "em_relaxed": 0.3515981735159817, + "em_relaxed_stderr": 0.03233829906905826 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0": { + "acc": 0.2706766917293233, + "acc_stderr": 0.027293707912941223, + "em_relaxed": 0.2706766917293233, + "em_relaxed_stderr": 0.027293707912941223 + }, + "e_tests.prompt1_mqa_multiple_correct_sk:matematika|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0": { + "acc": 0.27522935779816515, + "acc_stderr": 0.042976970405367555, + "em_relaxed": 0.28440366972477066, + "em_relaxed_stderr": 0.043409998674551635 + }, + "slovak_tests.prompt2_mqa_single_correct_sk:maturita|0": { + "acc": 0.2791327913279133, + "acc_stderr": 0.02338348056654487, + "em_relaxed": 0.2791327913279133, + "em_relaxed_stderr": 0.02338348056654487 + }, + "slovak_tests.prompt1_mqa_single_correct_sk:vyssie_studium|0": { + "acc": 0.26666666666666666, + "acc_stderr": 0.04336290903919941, + "em_relaxed": 0.26666666666666666, + "em_relaxed_stderr": 0.04336290903919941 + }, + "mmlu.prompt2_mqa_single_correct_sk:astronomy|0": { + "acc": 0.1513157894736842, + "acc_stderr": 0.029162631596843982, + "em_relaxed": 0.1513157894736842, + "em_relaxed_stderr": 0.029162631596843982 + }, + "mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0": { + "acc": 0.2828282828282828, + "acc_stderr": 0.04549461117008498, + "em_relaxed": 0.2828282828282828, + "em_relaxed_stderr": 0.04549461117008498 + }, + "e_tests.prompt3_mqa_single_correct_sk:biologia|0": { + "acc": 0.22988505747126436, + "acc_stderr": 0.045371581852507746, + "em_relaxed": 0.22988505747126436, + "em_relaxed_stderr": 0.045371581852507746 + }, + "e_tests.prompt1_qa_sk:chemia|0": { + "qem": 0.018518518518518517, + "qem_stderr": 0.018518518518518514, + "qf1": 0.012345679012345678, + "qf1_stderr": 0.012345679012345675, + "rouge1": 0.02580246913580247, + "rouge1_stderr": 0.011283149325161083, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.02580246913580247, + "rougeL_stderr": 0.011283149325161083 + }, + "e_tests.prompt2_qa_sk:biologia|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.019047619047619046, + "qf1_stderr": 0.019047619047619035, + "rouge1": 0.02857142857142857, + "rouge1_stderr": 0.01990372988626389, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.02857142857142857, + "rougeL_stderr": 0.01990372988626389 + }, + "mmlu.prompt2_mqa_single_correct_sk:business_ethics|0": { + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "em_relaxed": 0.3, + "em_relaxed_stderr": 0.046056618647183814 + }, + "mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0": { + "acc": 0.27, + "acc_stderr": 0.044619604333847394, + "em_relaxed": 0.27, + "em_relaxed_stderr": 0.044619604333847394 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0": { + "acc": 0.2, + "acc_stderr": 0.032769276820761584, + "em_relaxed": 0.20666666666666667, + "em_relaxed_stderr": 0.033171868444173455 + }, + "slovak_tests.prompt2_qa_sk:monitor9|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "scio.prompt2_mqa_single_correct_sk:mat|0": { + "acc": 0.23770491803278687, + "acc_stderr": 0.022280975973791882, + "em_relaxed": 0.24316939890710382, + "em_relaxed_stderr": 0.022454705846132374 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0": { + "acc": 0.18686868686868688, + "acc_stderr": 0.027772533334218974, + "em_relaxed": 0.1919191919191919, + "em_relaxed_stderr": 0.028057791672989024 + }, + "multi_wiki_qa.prompt1_qa_sk|0": { + "qem": 0.0756453872323394, + "qem_stderr": 0.003741099997048089, + "qf1": 0.12194744826062183, + "qf1_stderr": 0.0038908205119327767 + }, + "mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0": { + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "em_relaxed": 0.29, + "em_relaxed_stderr": 0.045604802157206845 + }, + "e_tests.prompt1_mqa_multiple_correct_sk:obcianska-nauka|0": { + "acc": 0.03571428571428571, + "acc_stderr": 0.03571428571428572 + }, + "mmlu.prompt1_mqa_single_correct_sk:security_studies|0": { + "acc": 0.18018018018018017, + "acc_stderr": 0.02585333527853839, + "em_relaxed": 0.18018018018018017, + "em_relaxed_stderr": 0.02585333527853839 + }, + "scio.prompt2_mqa_single_correct_sk:vsp|0": { + "acc": 0.25849514563106796, + "acc_stderr": 0.015261010021381638, + "em_relaxed": 0.2621359223300971, + "em_relaxed_stderr": 0.01533033114193304 + }, + "e_tests.prompt2_qa_sk:dejepis|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0": { + "acc": 0.16580310880829016, + "acc_stderr": 0.026839845022314415, + "em_relaxed": 0.16580310880829016, + "em_relaxed_stderr": 0.026839845022314415 + }, + "mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0": { + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "em_relaxed": 0.21, + "em_relaxed_stderr": 0.040936018074033256 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0": { + "acc": 0.23232323232323232, + "acc_stderr": 0.04266016017054689, + "em_relaxed": 0.24242424242424243, + "em_relaxed_stderr": 0.04329004329004326 + }, + "mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0": { + "acc": 0.2463768115942029, + "acc_stderr": 0.025984280443996907, + "em_relaxed": 0.2463768115942029, + "em_relaxed_stderr": 0.025984280443996907 + }, + "e_tests.prompt1_qa_sk:informatika|0": { + "qem": 0.038461538461538464, + "qem_stderr": 0.021915495473007307, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0032865282865282863, + "rouge1_stderr": 0.0022973226212271926, + "rouge2": 0.0008547008547008547, + "rouge2_stderr": 0.0008547008547008546, + "rougeL": 0.0032865282865282863, + "rougeL_stderr": 0.0022973226212271926 + }, + "e_tests.prompt3_mqa_multiple_correct_sk:matematika|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0": { + "acc": 0.18, + "acc_stderr": 0.038612291966536955, + "em_relaxed": 0.18, + "em_relaxed_stderr": 0.038612291966536955 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0": { + "acc": 0.22119815668202766, + "acc_stderr": 0.02824082382619082, + "em_relaxed": 0.2304147465437788, + "em_relaxed_stderr": 0.0286521112206854 + }, + "mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0": { + "acc": 0.22162883845126835, + "acc_stderr": 0.015186430423507605, + "em_relaxed": 0.22162883845126835, + "em_relaxed_stderr": 0.015186430423507605 + }, + "slovak_tests.prompt3_mqa_single_correct_sk:prijmacie_skusky|0": { + "acc": 0.24744897959183673, + "acc_stderr": 0.021823400829422215, + "em_relaxed": 0.24744897959183673, + "em_relaxed_stderr": 0.021823400829422215 + }, + "mmlu.prompt2_mqa_single_correct_sk:global_facts|0": { + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "em_relaxed": 0.3, + "em_relaxed_stderr": 0.046056618647183814 + }, + "truthfulqa.prompt2_mqa_single_correct_sk:mc1|0": { + "acc": 0.397172236503856, + "acc_stderr": 0.01755397820008727, + "em_relaxed": 0.40359897172236503, + "em_relaxed_stderr": 0.017600852681572653 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0": { + "acc": 0.24561403508771928, + "acc_stderr": 0.057521390564052664, + "em_relaxed": 0.24561403508771928, + "em_relaxed_stderr": 0.057521390564052664 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0": { + "acc": 0.21963824289405684, + "acc_stderr": 0.021072124180856616, + "em_relaxed": 0.21963824289405684, + "em_relaxed_stderr": 0.021072124180856616 + }, + "mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0": { + "acc": 0.2803738317757009, + "acc_stderr": 0.04362839933570099, + "em_relaxed": 0.2803738317757009, + "em_relaxed_stderr": 0.04362839933570099 + }, + "e_tests.prompt1_mqa_single_correct_sk:geografia|0": { + "acc": 0.24489795918367346, + "acc_stderr": 0.062069005411206336, + "em_relaxed": 0.24489795918367346, + "em_relaxed_stderr": 0.062069005411206336 + }, + "truthfulqa.prompt3_mqa_single_correct_sk:mc1|0": { + "acc": 0.13624678663239073, + "acc_stderr": 0.012306865740559735, + "em_relaxed": 0.14524421593830333, + "em_relaxed_stderr": 0.012640374409743442 + }, + "mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0": { + "acc": 0.21739130434782608, + "acc_stderr": 0.024872937110754584, + "em_relaxed": 0.21739130434782608, + "em_relaxed_stderr": 0.024872937110754584 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0": { + "acc": 0.24444444444444444, + "acc_stderr": 0.03212157057535212, + "em_relaxed": 0.24444444444444444, + "em_relaxed_stderr": 0.03212157057535212 + }, + "mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0": { + "acc": 0.22830440587449932, + "acc_stderr": 0.015347207245326627, + "em_relaxed": 0.22963951935914553, + "em_relaxed_stderr": 0.015378696047536682 + }, + "e_tests.prompt3_qa_sk:informatika|0": { + "qem": 0.02564102564102564, + "qem_stderr": 0.01801282495677459, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0036324786324786326, + "rouge1_stderr": 0.0025563439430967027, + "rouge2": 0.0008547008547008547, + "rouge2_stderr": 0.0008547008547008546, + "rougeL": 0.0036324786324786326, + "rougeL_stderr": 0.0025563439430967027 + }, + "mmlu.prompt2_mqa_single_correct_sk:philosophy|0": { + "acc": 0.21753246753246752, + "acc_stderr": 0.023546481379339165, + "em_relaxed": 0.21753246753246752, + "em_relaxed_stderr": 0.023546481379339165 + }, + "mmlu.prompt1_mqa_single_correct_sk:management|0": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "em_relaxed": 0.25, + "em_relaxed_stderr": 0.04351941398892446 + }, + "scio.prompt3_mqa_single_correct_sk:bio|0": { + "acc": 0.24778761061946902, + "acc_stderr": 0.023482897631940607, + "em_relaxed": 0.26548672566371684, + "em_relaxed_stderr": 0.024019440114877274 + }, + "mmlu.prompt1_mqa_single_correct_sk:public_relations|0": { + "acc": 0.30275229357798167, + "acc_stderr": 0.04421049631437605, + "em_relaxed": 0.30275229357798167, + "em_relaxed_stderr": 0.04421049631437605 + }, + "truthfulqa.prompt1_mqa_single_correct_sk:mc1|0": { + "acc": 0.42287917737789205, + "acc_stderr": 0.017722745992252577, + "em_relaxed": 0.42287917737789205, + "em_relaxed_stderr": 0.017722745992252577 + }, + "mmlu.prompt3_mqa_single_correct_sk:college_physics|0": { + "acc": 0.17647058823529413, + "acc_stderr": 0.0379328118530781, + "em_relaxed": 0.18627450980392157, + "em_relaxed_stderr": 0.03873958714149352 + }, + "financial_exams.prompt1_completion_sk|0": { + "acc": 0.18290854572713644, + "acc_stderr": 0.010588568678863373, + "em_relaxed": 0.18290854572713644, + "em_relaxed_stderr": 0.010588568678863373 + }, + "mmlu.prompt3_mqa_single_correct_sk:public_relations|0": { + "acc": 0.3211009174311927, + "acc_stderr": 0.044927422774799206, + "em_relaxed": 0.3211009174311927, + "em_relaxed_stderr": 0.044927422774799206 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0": { + "acc": 0.20666666666666667, + "acc_stderr": 0.033171868444173455, + "em_relaxed": 0.22, + "em_relaxed_stderr": 0.033936379482039564 + }, + "sklep.prompt3_hate_speech_sk:hate-speech|0": { + "acc": 0.0932524639878696, + "acc_stderr": 0.00800968883832857, + "em_norm_relaxed": 0.09401061410159212, + "em_norm_relaxed_stderr": 0.008038819818872478 + }, + "e_tests.prompt3_qa_sk:geografia|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "mmlu.prompt2_mqa_single_correct_sk:sociology|0": { + "acc": 0.233502538071066, + "acc_stderr": 0.030218515643241007, + "em_relaxed": 0.233502538071066, + "em_relaxed_stderr": 0.030218515643241007 + }, + "mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0": { + "acc": 0.21212121212121213, + "acc_stderr": 0.041296069325408906, + "em_relaxed": 0.21212121212121213, + "em_relaxed_stderr": 0.041296069325408906 + }, + "mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0": { + "acc": 0.23232323232323232, + "acc_stderr": 0.042660160170546885, + "em_relaxed": 0.23232323232323232, + "em_relaxed_stderr": 0.042660160170546885 + }, + "mmlu.prompt1_mqa_single_correct_sk:machine_learning|0": { + "acc": 0.26785714285714285, + "acc_stderr": 0.04203277291467762, + "em_relaxed": 0.2767857142857143, + "em_relaxed_stderr": 0.04246624336697624 + }, + "slovakcopa.prompt3_copa_sk|0": { + "acc": 0.016, + "acc_stderr": 0.0056170367814719815, + "em_relaxed": 0.016, + "em_relaxed_stderr": 0.0056170367814719815 + }, + "e_tests.prompt1_mqa_multiple_correct_sk:slovensky-jazyk|0": { + "acc": 0.02142857142857143, + "acc_stderr": 0.012282463299882659 + }, + "mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0": { + "acc": 0.27102803738317754, + "acc_stderr": 0.043172737765666686, + "em_relaxed": 0.27102803738317754, + "em_relaxed_stderr": 0.043172737765666686 + }, + "slovak_tests.prompt2_mqa_single_correct_sk:monitor9|0": { + "acc": 0.2903225806451613, + "acc_stderr": 0.030884745411494743, + "em_relaxed": 0.2903225806451613, + "em_relaxed_stderr": 0.030884745411494743 + }, + "financial_exams.prompt2_completion_sk|0": { + "acc": 0.1896551724137931, + "acc_stderr": 0.010737475929667924, + "em_relaxed": 0.1904047976011994, + "em_relaxed_stderr": 0.010753697898553788 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0": { + "acc": 0.19170984455958548, + "acc_stderr": 0.02840895362624528, + "em_relaxed": 0.19170984455958548, + "em_relaxed_stderr": 0.02840895362624528 + }, + "e_tests.prompt1_qa_sk:slovensky-jazyk|0": { + "qem": 0.0051813471502590676, + "qem_stderr": 0.00518134715025908, + "qf1": 0.01526314790563495, + "qf1_stderr": 0.006730183842906608, + "rouge1": 0.01724455022545386, + "rouge1_stderr": 0.006851608530741241, + "rouge2": 0.0073447868375602215, + "rouge2_stderr": 0.005570575389175516, + "rougeL": 0.01724455022545386, + "rougeL_stderr": 0.006851608530741241 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0": { + "acc": 0.22962962962962963, + "acc_stderr": 0.03633384414073462, + "em_relaxed": 0.22962962962962963, + "em_relaxed_stderr": 0.03633384414073462 + }, + "mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0": { + "acc": 0.2862644415917843, + "acc_stderr": 0.01620550507370852, + "em_relaxed": 0.28754813863928114, + "em_relaxed_stderr": 0.01622718713515171 + }, + "mmlu.prompt3_mqa_single_correct_sk:anatomy|0": { + "acc": 0.2537313432835821, + "acc_stderr": 0.037731914327133756, + "em_relaxed": 0.2537313432835821, + "em_relaxed_stderr": 0.037731914327133756 + }, + "sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0": { + "acc": 0.04702495201535509, + "acc_stderr": 0.006561144166938836, + "em_norm_relaxed": 0.05566218809980806, + "em_norm_relaxed_stderr": 0.007105889705516419 + }, + "toxic_sk.prompt3_toxicity_sk|0": { + "acc": 0.007945516458569807, + "acc_stderr": 0.0029928675111869866, + "em_norm_relaxed": 0.007945516458569807, + "em_norm_relaxed_stderr": 0.0029928675111869866 + }, + "e_tests.prompt3_mqa_multiple_correct_sk:biologia|0": { + "acc": 0.017857142857142856, + "acc_stderr": 0.017857142857142863 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0": { + "acc": 0.24561403508771928, + "acc_stderr": 0.05752139056405266, + "em_relaxed": 0.24561403508771928, + "em_relaxed_stderr": 0.05752139056405266 + }, + "slovak_tests.prompt3_qa_sk:maturita|0": { + "qem": 0.01327433628318584, + "qem_stderr": 0.007629803632081178, + "qf1": 0.011971484759095378, + "qf1_stderr": 0.0058019008534905304, + "rouge1": 0.014379696684114138, + "rouge1_stderr": 0.0046830153944730545, + "rouge2": 0.004080629301868239, + "rouge2_stderr": 0.002569082757498883, + "rougeL": 0.014379696684114138, + "rougeL_stderr": 0.0046830153944730545 + }, + "mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0": { + "acc": 0.2, + "acc_stderr": 0.04020151261036845, + "em_relaxed": 0.2, + "em_relaxed_stderr": 0.04020151261036845 + }, + "slovak_tests.prompt2_qa_sk:maturita|0": { + "qem": 0.008849557522123894, + "qem_stderr": 0.0062436609110668876, + "qf1": 0.0057030481809242865, + "qf1_stderr": 0.0035655859886822355, + "rouge1": 0.015324943300924803, + "rouge1_stderr": 0.0047498192758248054, + "rouge2": 0.0022123893805309734, + "rouge2_stderr": 0.002212389380530974, + "rougeL": 0.015324943300924803, + "rougeL_stderr": 0.0047498192758248054 + }, + "slovak_tests.prompt3_qa_sk:prijmacie_skusky|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.036387264457439894, + "qf1_stderr": 0.025966525279646633, + "rouge1": 0.027777777777777776, + "rouge1_stderr": 0.019641855032959652, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.027777777777777776, + "rougeL_stderr": 0.019641855032959652 + }, + "e_tests.prompt2_mqa_single_correct_sk:dejepis|0": { + "acc": 0.1875, + "acc_stderr": 0.05693291124629694, + "em_relaxed": 0.1875, + "em_relaxed_stderr": 0.05693291124629694 + }, + "slovak_tests.prompt2_mqa_single_correct_sk:prijmacie_skusky|0": { + "acc": 0.2653061224489796, + "acc_stderr": 0.022327416188566384, + "em_relaxed": 0.2653061224489796, + "em_relaxed_stderr": 0.022327416188566384 + }, + "e_tests.prompt2_qa_sk:chemia|0": { + "qem": 0.037037037037037035, + "qem_stderr": 0.025940896147128693, + "qf1": 0.012345679012345678, + "qf1_stderr": 0.012345679012345675, + "rouge1": 0.02580246913580247, + "rouge1_stderr": 0.011283149325161083, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.02580246913580247, + "rougeL_stderr": 0.011283149325161083 + }, + "e_tests.prompt2_mqa_single_correct_sk:slovensky-jazyk|0": { + "acc": 0.26419213973799127, + "acc_stderr": 0.020624529862085718, + "em_relaxed": 0.26419213973799127, + "em_relaxed_stderr": 0.020624529862085718 + }, + "mmlu.prompt3_mqa_single_correct_sk:philosophy|0": { + "acc": 0.22402597402597402, + "acc_stderr": 0.02379597980086175, + "em_relaxed": 0.22402597402597402, + "em_relaxed_stderr": 0.02379597980086175 + }, + "e_tests.prompt1_mqa_single_correct_sk:fyzika|0": { + "acc": 0.22727272727272727, + "acc_stderr": 0.0914486154730632, + "em_relaxed": 0.22727272727272727, + "em_relaxed_stderr": 0.0914486154730632 + }, + "mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0": { + "acc": 0.19318181818181818, + "acc_stderr": 0.024344064423295675, + "em_relaxed": 0.19318181818181818, + "em_relaxed_stderr": 0.024344064423295675 + }, + "e_tests.prompt3_mqa_multiple_correct_sk:fyzika|0": { + "acc": 0.1, + "acc_stderr": 0.06882472016116853 + }, + "mmlu.prompt2_mqa_single_correct_sk:computer_security|0": { + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "em_relaxed": 0.3, + "em_relaxed_stderr": 0.046056618647183814 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0": { + "acc": 0.24812030075187969, + "acc_stderr": 0.02653275776026344, + "em_relaxed": 0.2518796992481203, + "em_relaxed_stderr": 0.026666091559782525 + }, + "mmlu.prompt2_mqa_single_correct_sk:college_biology|0": { + "acc": 0.2605633802816901, + "acc_stderr": 0.03696558732615874, + "em_relaxed": 0.2605633802816901, + "em_relaxed_stderr": 0.03696558732615874 + }, + "e_tests.prompt1_mqa_multiple_correct_sk:informatika|0": { + "acc": 0.02857142857142857, + "acc_stderr": 0.028571428571428574 + }, + "slovak_tests.prompt2_qa_sk:prijmacie_skusky|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.024691358024691357, + "qf1_stderr": 0.024691358024691353, + "rouge1": 0.01851851851851852, + "rouge1_stderr": 0.01851851851851852, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.01851851851851852, + "rougeL_stderr": 0.01851851851851852 + }, + "mmlu.prompt1_mqa_single_correct_sk:anatomy|0": { + "acc": 0.23134328358208955, + "acc_stderr": 0.036565274697599, + "em_relaxed": 0.23134328358208955, + "em_relaxed_stderr": 0.036565274697599 + }, + "mmlu.prompt3_mqa_single_correct_sk:management|0": { + "acc": 0.26, + "acc_stderr": 0.04408440022768078, + "em_relaxed": 0.26, + "em_relaxed_stderr": 0.04408440022768078 + }, + "mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0": { + "acc": 0.2088607594936709, + "acc_stderr": 0.03244180901572427, + "em_relaxed": 0.2088607594936709, + "em_relaxed_stderr": 0.03244180901572427 + }, + "mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0": { + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "em_relaxed": 0.31, + "em_relaxed_stderr": 0.04648231987117316 + }, + "scio.prompt2_mqa_single_correct_sk:aj|0": { + "acc": 0.3, + "acc_stderr": 0.15275252316519464, + "em_relaxed": 0.3, + "em_relaxed_stderr": 0.15275252316519464 + }, + "mmlu.prompt2_mqa_single_correct_sk:formal_logic|0": { + "acc": 0.21739130434782608, + "acc_stderr": 0.038631448549506, + "em_relaxed": 0.25217391304347825, + "em_relaxed_stderr": 0.04067222754154718 + }, + "mmlu.prompt2_mqa_single_correct_sk:international_law|0": { + "acc": 0.2457627118644068, + "acc_stderr": 0.03980329854920433, + "em_relaxed": 0.2457627118644068, + "em_relaxed_stderr": 0.03980329854920433 + }, + "e_tests.prompt1_mqa_single_correct_sk:chemia|0": { + "acc": 0.17857142857142858, + "acc_stderr": 0.05164277182008721, + "em_relaxed": 0.17857142857142858, + "em_relaxed_stderr": 0.05164277182008721 + }, + "mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0": { + "acc": 0.2386058981233244, + "acc_stderr": 0.022099063590345355, + "em_relaxed": 0.2386058981233244, + "em_relaxed_stderr": 0.022099063590345355 + }, + "e_tests.prompt3_mqa_single_correct_sk:slovensky-jazyk|0": { + "acc": 0.27510917030567683, + "acc_stderr": 0.020889629452003455, + "em_relaxed": 0.27510917030567683, + "em_relaxed_stderr": 0.020889629452003455 + }, + "mmlu.prompt3_mqa_single_correct_sk:astronomy|0": { + "acc": 0.17763157894736842, + "acc_stderr": 0.03110318238312337, + "em_relaxed": 0.18421052631578946, + "em_relaxed_stderr": 0.031546980450822305 + }, + "sklep.prompt1_hate_speech_sk:hate-speech|0": { + "acc": 0.3525398028809704, + "acc_stderr": 0.01315990975593033, + "em_norm_relaxed": 0.35329795299469297, + "em_norm_relaxed_stderr": 0.013166337192115686 + }, + "e_tests.prompt1_mqa_multiple_correct_sk:dejepis|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "mmlu.prompt2_mqa_single_correct_sk:anatomy|0": { + "acc": 0.21641791044776118, + "acc_stderr": 0.035707794739409195, + "em_relaxed": 0.22388059701492538, + "em_relaxed_stderr": 0.03614487092173135 + }, + "e_tests.prompt1_mqa_single_correct_sk:matematika|0": { + "acc": 0.1282051282051282, + "acc_stderr": 0.05423355275954149, + "em_relaxed": 0.1282051282051282, + "em_relaxed_stderr": 0.05423355275954149 + }, + "sklep.prompt3_qa_sk:question-answering|0": { + "qem": 0.05123656475007826, + "qem_stderr": 0.0022523759848186195, + "qf1": 0.1417299858339243, + "qf1_stderr": 0.002706574024592746 + }, + "e_tests.prompt1_mqa_multiple_correct_sk:chemia|0": { + "acc": 0.022727272727272728, + "acc_stderr": 0.022727272727272728 + }, + "mmlu.prompt1_mqa_single_correct_sk:philosophy|0": { + "acc": 0.2077922077922078, + "acc_stderr": 0.023156077580247444, + "em_relaxed": 0.2077922077922078, + "em_relaxed_stderr": 0.023156077580247444 + }, + "toxic_sk.prompt2_toxicity_sk|0": { + "acc": 0.23609534619750283, + "acc_stderr": 0.014316008748210712, + "em_norm_relaxed": 0.23723041997729852, + "em_norm_relaxed_stderr": 0.014339715502704882 + }, + "mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0": { + "acc": 0.24770642201834864, + "acc_stderr": 0.041538463936410155, + "em_relaxed": 0.24770642201834864, + "em_relaxed_stderr": 0.041538463936410155 + }, + "e_tests.prompt3_mqa_multiple_correct_sk:obcianska-nauka|0": { + "acc": 0.03571428571428571, + "acc_stderr": 0.03571428571428572 + }, + "senti_sk.prompt3_sentiment_binary_sk|0": { + "acc": 0.7416845493562232, + "acc_stderr": 0.007169768138157722, + "em_norm_relaxed": 0.7475858369098712, + "em_norm_relaxed_stderr": 0.007115537241408355 + }, + "mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0": { + "acc": 0.22758620689655173, + "acc_stderr": 0.03493950380131184, + "em_relaxed": 0.22758620689655173, + "em_relaxed_stderr": 0.03493950380131184 + }, + "mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0": { + "acc": 0.1856060606060606, + "acc_stderr": 0.023973722216817445, + "em_relaxed": 0.1856060606060606, + "em_relaxed_stderr": 0.023973722216817445 + }, + "e_tests.prompt2_mqa_single_correct_sk:informatika|0": { + "acc": 0.232, + "acc_stderr": 0.03790650637827825, + "em_relaxed": 0.232, + "em_relaxed_stderr": 0.03790650637827825 + }, + "mmlu.prompt2_mqa_single_correct_sk:management|0": { + "acc": 0.27, + "acc_stderr": 0.044619604333847394, + "em_relaxed": 0.27, + "em_relaxed_stderr": 0.044619604333847394 + }, + "mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0": { + "acc": 0.27, + "acc_stderr": 0.044619604333847394, + "em_relaxed": 0.27, + "em_relaxed_stderr": 0.044619604333847394 + }, + "toxic_sk.prompt1_toxicity_sk|0": { + "acc": 0.3530079455164586, + "acc_stderr": 0.01611017944424598, + "em_norm_relaxed": 0.35414301929625425, + "em_norm_relaxed_stderr": 0.016121898615772852 + }, + "mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0": { + "acc": 0.2651006711409396, + "acc_stderr": 0.018095094525450207, + "em_relaxed": 0.26677852348993286, + "em_relaxed_stderr": 0.018131533555282176 + }, + "mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0": { + "acc": 0.23448275862068965, + "acc_stderr": 0.035306258743465914, + "em_relaxed": 0.23448275862068965, + "em_relaxed_stderr": 0.035306258743465914 + }, + "sklep.prompt2_hate_speech_sk:hate-speech|0": { + "acc": 0.1956027293404094, + "acc_stderr": 0.010926096810556464, + "em_norm_relaxed": 0.1956027293404094, + "em_norm_relaxed_stderr": 0.010926096810556464 + }, + "scio.prompt3_mqa_single_correct_sk:che|0": { + "acc": 0.24796747967479674, + "acc_stderr": 0.027588788664854164, + "em_relaxed": 0.24796747967479674, + "em_relaxed_stderr": 0.027588788664854164 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0": { + "acc": 0.21717171717171718, + "acc_stderr": 0.029376616484945637, + "em_relaxed": 0.21717171717171718, + "em_relaxed_stderr": 0.029376616484945637 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0": { + "acc": 0.2222222222222222, + "acc_stderr": 0.031073844843659416, + "em_relaxed": 0.2222222222222222, + "em_relaxed_stderr": 0.031073844843659416 + }, + "e_tests.prompt1_mqa_single_correct_sk:biologia|0": { + "acc": 0.3103448275862069, + "acc_stderr": 0.04988718850038745, + "em_relaxed": 0.3103448275862069, + "em_relaxed_stderr": 0.04988718850038745 + }, + "e_tests.prompt2_mqa_multiple_correct_sk:dejepis|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "e_tests.prompt3_mqa_single_correct_sk:chemia|0": { + "acc": 0.23214285714285715, + "acc_stderr": 0.05692939024000109, + "em_relaxed": 0.23214285714285715, + "em_relaxed_stderr": 0.05692939024000109 + }, + "e_tests.prompt2_mqa_single_correct_sk:chemia|0": { + "acc": 0.21428571428571427, + "acc_stderr": 0.055328333517248834, + "em_relaxed": 0.21428571428571427, + "em_relaxed_stderr": 0.055328333517248834 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0": { + "acc": 0.17616580310880828, + "acc_stderr": 0.027493504244548047, + "em_relaxed": 0.17616580310880828, + "em_relaxed_stderr": 0.027493504244548047 + }, + "e_tests.prompt3_mqa_single_correct_sk:matematika|0": { + "acc": 0.20512820512820512, + "acc_stderr": 0.06550424345215436, + "em_relaxed": 0.20512820512820512, + "em_relaxed_stderr": 0.06550424345215436 + }, + "e_tests.prompt3_qa_sk:obcianska-nauka|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.020408163265306124, + "rouge1_stderr": 0.020408163265306124, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.020408163265306124, + "rougeL_stderr": 0.020408163265306124 + }, + "e_tests.prompt3_mqa_single_correct_sk:dejepis|0": { + "acc": 0.14583333333333334, + "acc_stderr": 0.05148145136372286, + "em_relaxed": 0.14583333333333334, + "em_relaxed_stderr": 0.05148145136372286 + }, + "e_tests.prompt3_mqa_multiple_correct_sk:chemia|0": { + "acc": 0.022727272727272728, + "acc_stderr": 0.022727272727272728 + }, + "mmlu.prompt3_mqa_single_correct_sk:professional_law|0": { + "acc": 0.25704526046114434, + "acc_stderr": 0.012775945430940199, + "em_relaxed": 0.2681468830059778, + "em_relaxed_stderr": 0.012951062793076921 + }, + "sklep.prompt2_nli_3_classes_sk:nli|0": { + "acc": 0.04936051159072742, + "acc_stderr": 0.003062545148658699, + "em_norm_relaxed": 0.06474820143884892, + "em_norm_relaxed_stderr": 0.003479067694461015 + }, + "mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0": { + "acc": 0.2088607594936709, + "acc_stderr": 0.03244180901572427, + "em_relaxed": 0.2088607594936709, + "em_relaxed_stderr": 0.03244180901572427 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0": { + "acc": 0.2332089552238806, + "acc_stderr": 0.01828244336455249, + "em_relaxed": 0.23507462686567165, + "em_relaxed_stderr": 0.01833308365747139 + }, + "mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0": { + "acc": 0.26605504587155965, + "acc_stderr": 0.042521210223474465, + "em_relaxed": 0.27522935779816515, + "em_relaxed_stderr": 0.04297697040536756 + }, + "senti_sk.prompt2_sentiment_3_class_sk|0": { + "acc": 0.3158007269617276, + "acc_stderr": 0.006797678423770271, + "em_norm_relaxed": 0.32050459696386574, + "em_norm_relaxed_stderr": 0.006824536198921628 + }, + "slovak_tests.prompt1_qa_sk:monitor9|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0": { + "acc": 0.20833333333333334, + "acc_stderr": 0.02504221193381759, + "em_relaxed": 0.20833333333333334, + "em_relaxed_stderr": 0.02504221193381759 + }, + "mmlu.prompt2_mqa_single_correct_sk:machine_learning|0": { + "acc": 0.24107142857142858, + "acc_stderr": 0.04059867246952688, + "em_relaxed": 0.25892857142857145, + "em_relaxed_stderr": 0.041577515398656284 + }, + "mmlu.prompt1_mqa_single_correct_sk:college_biology|0": { + "acc": 0.24647887323943662, + "acc_stderr": 0.03629342967383697, + "em_relaxed": 0.24647887323943662, + "em_relaxed_stderr": 0.03629342967383697 + }, + "e_tests.prompt3_qa_sk:biologia|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.019047619047619046, + "qf1_stderr": 0.019047619047619035, + "rouge1": 0.014285714285714285, + "rouge1_stderr": 0.014285714285714287, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.014285714285714285, + "rougeL_stderr": 0.014285714285714287 + }, + "mmlu.prompt3_mqa_single_correct_sk:prehistory|0": { + "acc": 0.25705329153605017, + "acc_stderr": 0.02450624980268527, + "em_relaxed": 0.26332288401253917, + "em_relaxed_stderr": 0.024698429072332967 + }, + "e_tests.prompt3_mqa_single_correct_sk:fyzika|0": { + "acc": 0.18181818181818182, + "acc_stderr": 0.08416546361568647, + "em_relaxed": 0.18181818181818182, + "em_relaxed_stderr": 0.08416546361568647 + }, + "senti_sk.prompt1_sentiment_3_class_sk|0": { + "acc": 0.25486422920675644, + "acc_stderr": 0.006372867628034562, + "em_norm_relaxed": 0.2572161642078255, + "em_norm_relaxed_stderr": 0.00639209321447505 + }, + "mmlu.prompt1_mqa_single_correct_sk:business_ethics|0": { + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "em_relaxed": 0.3, + "em_relaxed_stderr": 0.046056618647183814 + }, + "mmlu.prompt1_mqa_single_correct_sk:college_physics|0": { + "acc": 0.16666666666666666, + "acc_stderr": 0.03708284662416542, + "em_relaxed": 0.16666666666666666, + "em_relaxed_stderr": 0.03708284662416542 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0": { + "acc": 0.25806451612903225, + "acc_stderr": 0.029772845386803175, + "em_relaxed": 0.25806451612903225, + "em_relaxed_stderr": 0.029772845386803175 + }, + "scio.prompt2_mqa_single_correct_sk:bio|0": { + "acc": 0.23008849557522124, + "acc_stderr": 0.02289335836732507, + "em_relaxed": 0.2448377581120944, + "em_relaxed_stderr": 0.02338842519033371 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0": { + "acc": 0.18691588785046728, + "acc_stderr": 0.02671167024582658, + "em_relaxed": 0.19158878504672897, + "em_relaxed_stderr": 0.026965681762486853 + }, + "mmlu.prompt3_mqa_single_correct_sk:international_law|0": { + "acc": 0.2711864406779661, + "acc_stderr": 0.041100705493392085, + "em_relaxed": 0.2711864406779661, + "em_relaxed_stderr": 0.041100705493392085 + }, + "e_tests.prompt2_mqa_single_correct_sk:obcianska-nauka|0": { + "acc": 0.18840579710144928, + "acc_stderr": 0.047420064740574225, + "em_relaxed": 0.18840579710144928, + "em_relaxed_stderr": 0.047420064740574225 + }, + "mmlu.prompt2_mqa_single_correct_sk:nutrition|0": { + "acc": 0.24092409240924093, + "acc_stderr": 0.0246081660183802, + "em_relaxed": 0.24752475247524752, + "em_relaxed_stderr": 0.0248343018351657 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0": { + "acc": 0.2727272727272727, + "acc_stderr": 0.03173071239071724, + "em_relaxed": 0.2727272727272727, + "em_relaxed_stderr": 0.03173071239071724 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0": { + "acc": 0.23255813953488372, + "acc_stderr": 0.021502792400356758, + "em_relaxed": 0.23255813953488372, + "em_relaxed_stderr": 0.021502792400356758 + }, + "scio.prompt1_mqa_single_correct_sk:che|0": { + "acc": 0.27235772357723576, + "acc_stderr": 0.028441059280558723, + "em_relaxed": 0.27235772357723576, + "em_relaxed_stderr": 0.028441059280558723 + }, + "mmlu.prompt1_mqa_single_correct_sk:college_medicine|0": { + "acc": 0.18604651162790697, + "acc_stderr": 0.029758600618213775, + "em_relaxed": 0.18604651162790697, + "em_relaxed_stderr": 0.029758600618213775 + }, + "slovak_tests.prompt3_qa_sk:monitor9|0": { + "qem": 0.0, + "qem_stderr": 0.0, + "qf1": 0.0, + "qf1_stderr": 0.0, + "rouge1": 0.0, + "rouge1_stderr": 0.0, + "rouge2": 0.0, + "rouge2_stderr": 0.0, + "rougeL": 0.0, + "rougeL_stderr": 0.0 + }, + "mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0": { + "acc": 0.21481481481481482, + "acc_stderr": 0.03547854198560827, + "em_relaxed": 0.21481481481481482, + "em_relaxed_stderr": 0.03547854198560827 + }, + "mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0": { + "acc": 0.19696969696969696, + "acc_stderr": 0.028335609732463348, + "em_relaxed": 0.19696969696969696, + "em_relaxed_stderr": 0.028335609732463348 + }, + "e_tests.prompt1_mqa_single_correct_sk:informatika|0": { + "acc": 0.24, + "acc_stderr": 0.03835319728439192, + "em_relaxed": 0.24, + "em_relaxed_stderr": 0.03835319728439192 + }, + "mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0": { + "acc": 0.28085106382978725, + "acc_stderr": 0.029379170464124825, + "em_relaxed": 0.28085106382978725, + "em_relaxed_stderr": 0.029379170464124825 + }, + "mmlu.prompt1_mqa_single_correct_sk:nutrition|0": { + "acc": 0.23102310231023102, + "acc_stderr": 0.02425386079112883, + "em_relaxed": 0.23102310231023102, + "em_relaxed_stderr": 0.02425386079112883 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0": { + "acc": 0.22695035460992907, + "acc_stderr": 0.024987106365642973, + "em_relaxed": 0.23049645390070922, + "em_relaxed_stderr": 0.025123739226872405 + }, + "mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0": { + "acc": 0.23333333333333334, + "acc_stderr": 0.03161296043612007, + "em_relaxed": 0.23333333333333334, + "em_relaxed_stderr": 0.03161296043612007 + }, + "sklep.prompt1_nli_binary_sk:rte|0": { + "acc": 0.26385542168674697, + "acc_stderr": 0.010820351700856653, + "em_norm_relaxed": 0.26626506024096386, + "em_norm_relaxed_stderr": 0.01085184289579353 + }, + "slovak_tests.prompt1_mqa_single_correct_sk:prijmacie_skusky|0": { + "acc": 0.22959183673469388, + "acc_stderr": 0.02126915716826347, + "em_relaxed": 0.22959183673469388, + "em_relaxed_stderr": 0.02126915716826347 + }, + "mmlu.prompt2_mqa_single_correct_sk:public_relations|0": { + "acc": 0.3486238532110092, + "acc_stderr": 0.04585456701913343, + "em_relaxed": 0.3486238532110092, + "em_relaxed_stderr": 0.04585456701913343 + }, + "slohry.prompt1_inflection_sk:inflection_low|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt1_declension_sk:declension_high|0": { + "acc": 0.0748502994011976, + "acc_stderr": 0.014420511342862789, + "em_norm_relaxed": 0.0748502994011976, + "em_norm_relaxed_stderr": 0.014420511342862789 + }, + "slohry.prompt3_gradation_sk:gradation_high|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt2_inflection_sk:inflection_high|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt2_gradation_sk:gradation_mid|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt2_declension_sk:declension_high|0": { + "acc": 0.059880239520958084, + "acc_stderr": 0.013002032231530652, + "em_norm_relaxed": 0.059880239520958084, + "em_norm_relaxed_stderr": 0.013002032231530652 + }, + "chiby.prompt2_chiby_sk|0": { + "acc": 0.025349650349650348, + "acc_stderr": 0.0026834904197041206, + "em_norm_relaxed": 0.02564102564102564, + "em_norm_relaxed_stderr": 0.002698465265697882 + }, + "slovak_trivia.prompt3_mqa_single_correct_sk|0": { + "acc": 0.28611898016997167, + "acc_stderr": 0.024088786371538086, + "em_relaxed": 0.28611898016997167, + "em_relaxed_stderr": 0.024088786371538086 + }, + "slohry.prompt1_inflection_sk:inflection_mid|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt1_gradation_sk:gradation_low|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "chiby.prompt3_chiby_sk|0": { + "acc": 0.1622960372960373, + "acc_stderr": 0.00629490070363013, + "em_norm_relaxed": 0.16258741258741258, + "em_norm_relaxed_stderr": 0.00629945304550759 + }, + "slohry.prompt2_declension_sk:declension_mid|0": { + "acc": 0.05389221556886228, + "acc_stderr": 0.012374031301903447, + "em_norm_relaxed": 0.05389221556886228, + "em_norm_relaxed_stderr": 0.012374031301903447 + }, + "slohry.prompt2_gradation_sk:gradation_low|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt2_gradation_sk:gradation_high|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt2_identity_form_sk:identity_form_low|0": { + "acc": 0.005988023952095809, + "acc_stderr": 0.004227809946139441, + "em_norm_relaxed": 0.005988023952095809, + "em_norm_relaxed_stderr": 0.004227809946139441 + }, + "slohry.prompt3_declension_sk:declension_high|0": { + "acc": 0.0718562874251497, + "acc_stderr": 0.014152002095505148, + "em_norm_relaxed": 0.0718562874251497, + "em_norm_relaxed_stderr": 0.014152002095505148 + }, + "slohry.prompt3_declension_sk:declension_mid|0": { + "acc": 0.0658682634730539, + "acc_stderr": 0.013593148263003514, + "em_norm_relaxed": 0.0658682634730539, + "em_norm_relaxed_stderr": 0.013593148263003514 + }, + "slohry.prompt1_identity_form_sk:identity_form_high|0": { + "acc": 0.0029940119760479044, + "acc_stderr": 0.0029940119760478922, + "em_norm_relaxed": 0.0029940119760479044, + "em_norm_relaxed_stderr": 0.0029940119760478922 + }, + "slohry.prompt3_declension_sk:declension_low|0": { + "acc": 0.04790419161676647, + "acc_stderr": 0.011703209185183546, + "em_norm_relaxed": 0.04790419161676647, + "em_norm_relaxed_stderr": 0.011703209185183546 + }, + "slohry.prompt1_declension_sk:declension_mid|0": { + "acc": 0.0718562874251497, + "acc_stderr": 0.014152002095505158, + "em_norm_relaxed": 0.0718562874251497, + "em_norm_relaxed_stderr": 0.014152002095505158 + }, + "slohry.prompt3_inflection_sk:inflection_high|0": { + "acc": 0.008982035928143712, + "acc_stderr": 0.005170184492247012, + "em_norm_relaxed": 0.008982035928143712, + "em_norm_relaxed_stderr": 0.005170184492247012 + }, + "slohry.prompt2_declension_sk:declension_low|0": { + "acc": 0.029940119760479042, + "acc_stderr": 0.00933907625586894, + "em_norm_relaxed": 0.029940119760479042, + "em_norm_relaxed_stderr": 0.00933907625586894 + }, + "slohry.prompt3_gradation_sk:gradation_mid|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt1_identity_form_sk:identity_form_low|0": { + "acc": 0.008982035928143712, + "acc_stderr": 0.005170184492247015, + "em_norm_relaxed": 0.008982035928143712, + "em_norm_relaxed_stderr": 0.005170184492247015 + }, + "slovak_trivia.prompt1_mqa_single_correct_sk|0": { + "acc": 0.26628895184135976, + "acc_stderr": 0.02355959235547647, + "em_relaxed": 0.26628895184135976, + "em_relaxed_stderr": 0.02355959235547647 + }, + "analogies_sk.prompt3_analogy_sk|0": { + "acc": 0.0, + "acc_stderr": 0.0 + }, + "slohry.prompt3_gradation_sk:gradation_low|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt3_inflection_sk:inflection_mid|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt1_inflection_sk:inflection_high|0": { + "acc": 0.0029940119760479044, + "acc_stderr": 0.0029940119760478966, + "em_norm_relaxed": 0.0029940119760479044, + "em_norm_relaxed_stderr": 0.0029940119760478966 + }, + "slohry.prompt3_inflection_sk:inflection_low|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt2_identity_form_sk:identity_form_high|0": { + "acc": 0.005988023952095809, + "acc_stderr": 0.004227809946139438, + "em_norm_relaxed": 0.005988023952095809, + "em_norm_relaxed_stderr": 0.004227809946139438 + }, + "slohry.prompt2_identity_form_sk:identity_form_mid|0": { + "acc": 0.02694610778443114, + "acc_stderr": 0.008873487700057157, + "em_norm_relaxed": 0.02694610778443114, + "em_norm_relaxed_stderr": 0.008873487700057157 + }, + "slohry.prompt3_identity_form_sk:identity_form_mid|0": { + "acc": 0.008982035928143712, + "acc_stderr": 0.0051701844922470075, + "em_norm_relaxed": 0.008982035928143712, + "em_norm_relaxed_stderr": 0.0051701844922470075 + }, + "slohry.prompt3_identity_form_sk:identity_form_low|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt1_declension_sk:declension_low|0": { + "acc": 0.05089820359281437, + "acc_stderr": 0.012044409374838807, + "em_norm_relaxed": 0.05089820359281437, + "em_norm_relaxed_stderr": 0.012044409374838807 + }, + "slohry.prompt1_gradation_sk:gradation_mid|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "analogies_sk.prompt2_analogy_sk|0": { + "acc": 0.0007544322897019992, + "acc_stderr": 0.0005333635248765946 + }, + "analogies_sk.prompt1_analogy_sk|0": { + "acc": 0.001131648434552999, + "acc_stderr": 0.0006531109317700804 + }, + "slohry.prompt1_gradation_sk:gradation_high|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt3_identity_form_sk:identity_form_high|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt2_inflection_sk:inflection_mid|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "slohry.prompt2_inflection_sk:inflection_low|0": { + "acc": 0.0, + "acc_stderr": 0.0, + "em_norm_relaxed": 0.0, + "em_norm_relaxed_stderr": 0.0 + }, + "chiby.prompt1_chiby_sk|0": { + "acc": 0.23572261072261072, + "acc_stderr": 0.00724629095400738, + "em_norm_relaxed": 0.23892773892773891, + "em_norm_relaxed_stderr": 0.007280075348854293 + }, + "slovak_trivia.prompt2_mqa_single_correct_sk|0": { + "acc": 0.28328611898017, + "acc_stderr": 0.024016749232030024, + "em_relaxed": 0.28328611898017, + "em_relaxed_stderr": 0.024016749232030024 + }, + "slohry.prompt1_identity_form_sk:identity_form_mid|0": { + "acc": 0.011976047904191617, + "acc_stderr": 0.005960989845848331, + "em_norm_relaxed": 0.011976047904191617, + "em_norm_relaxed_stderr": 0.005960989845848331 + }, + "movie_review_SK_AVERAGE|0": { + "em_norm_relaxed": 0.14890438247011953, + "acc": 0.1457503320053121, + "em_norm_relaxed_stderr": 0.007823268108665498, + "acc_stderr": 0.007746689156547597, + "em_norm_relaxed__prompts": { + "1": 0.10109561752988047, + "3": 0.20916334661354583, + "2": 0.13645418326693226 + }, + "acc__prompts": { + "1": 0.09860557768924302, + "3": 0.20766932270916336, + "2": 0.13097609561752988 + } + }, + "movie_review_AVERAGE|0": { + "em_norm_relaxed": 0.14890438247011953, + "em_norm_relaxed_stderr": 0.007823268108665498, + "acc": 0.1457503320053121, + "acc_stderr": 0.007746689156547597 + }, + "e_tests_mqa_SK_AVERAGE|0": { + "em_relaxed": 0.24309199020636585, + "acc": 0.18150087260034903, + "em_relaxed_stderr": 0.03621678600189384, + "acc_stderr": 0.03214832055891622, + "em_relaxed__prompts": { + "2": 0.24239244491080797, + "1": 0.2497376705141658, + "3": 0.23714585519412382 + }, + "acc__prompts": { + "2": 0.18175018698578907, + "1": 0.1862378459237098, + "3": 0.17651458489154825 + } + }, + "e_tests_mqa_AVERAGE|0": { + "em_relaxed": 0.24309199020636585, + "em_relaxed_stderr": 0.03621678600189384, + "acc": 0.18150087260034903, + "acc_stderr": 0.03214832055891622 + }, + "mmlu_SK_AVERAGE|0": { + "em_relaxed": 0.24332669831062115, + "acc": 0.24148930740570612, + "em_relaxed_stderr": 0.0266013460946822, + "acc_stderr": 0.026529494985304103, + "em_relaxed__prompts": { + "2": 0.24628693921298422, + "3": 0.2474353085285561, + "1": 0.23625784719032308 + }, + "acc__prompts": { + "2": 0.2440667585362119, + "3": 0.24437299035369775, + "1": 0.23602817332720868 + } + }, + "mmlu_AVERAGE|0": { + "em_relaxed": 0.24332669831062115, + "em_relaxed_stderr": 0.0266013460946822, + "acc": 0.24148930740570612, + "acc_stderr": 0.026529494985304103 + }, + "belebele_SK_AVERAGE|0": { + "em_relaxed": 0.267037037037037, + "acc": 0.26666666666666666, + "em_relaxed_stderr": 0.014753176443555857, + "acc_stderr": 0.014746835660938531, + "em_relaxed__prompts": { + "3": 0.2722222222222222, + "1": 0.2577777777777778, + "2": 0.27111111111111114 + }, + "acc__prompts": { + "3": 0.27111111111111114, + "1": 0.2577777777777778, + "2": 0.27111111111111114 + } + }, + "belebele_AVERAGE|0": { + "em_relaxed": 0.267037037037037, + "em_relaxed_stderr": 0.014753176443555857, + "acc": 0.26666666666666666, + "acc_stderr": 0.014746835660938531 + }, + "senti_sk_binary_SK_AVERAGE|0": { + "em_norm_relaxed": 0.49356223175965663, + "acc": 0.4873927038626609, + "em_norm_relaxed_stderr": 0.007613926970563059, + "acc_stderr": 0.0076136774621452415, + "em_norm_relaxed__prompts": { + "2": 0.3261802575107296, + "1": 0.4069206008583691, + "3": 0.7475858369098712 + }, + "acc__prompts": { + "2": 0.32135193133047213, + "1": 0.39914163090128757, + "3": 0.7416845493562232 + } + }, + "senti_sk_binary_AVERAGE|0": { + "em_norm_relaxed": 0.49356223175965663, + "em_norm_relaxed_stderr": 0.007613926970563059, + "acc": 0.4873927038626609, + "acc_stderr": 0.0076136774621452415 + }, + "slovak_tests_mqa_SK_AVERAGE|0": { + "em_relaxed": 0.2600800246229609, + "acc": 0.2600800246229609, + "em_relaxed_stderr": 0.02573346982142187, + "acc_stderr": 0.02573346982142187, + "em_relaxed__prompts": { + "2": 0.26685133887349954, + "3": 0.25761772853185594, + "1": 0.25577100646352724 + }, + "acc__prompts": { + "2": 0.26685133887349954, + "3": 0.25761772853185594, + "1": 0.25577100646352724 + } + }, + "slovak_tests_mqa_AVERAGE|0": { + "em_relaxed": 0.2600800246229609, + "em_relaxed_stderr": 0.02573346982142187, + "acc": 0.2600800246229609, + "acc_stderr": 0.02573346982142187 + }, + "e_tests_qa_SK_AVERAGE|0": { + "rouge2": 0.003228194982976972, + "rouge1": 0.014818769352489746, + "rougeL": 0.014818769352489746, + "qf1": 0.007878222192390297, + "qem": 0.012039127163280662, + "rouge2_stderr": 0.002547306056000719, + "rouge1_stderr": 0.008024257279824093, + "rougeL_stderr": 0.008024257279824093, + "qf1_stderr": 0.005529975861330393, + "qem_stderr": 0.00874750521016363, + "rouge2__prompts": { + "3": 0.0030234626171420755, + "2": 0.003310759969902182, + "1": 0.0033503623618866578 + }, + "rouge1__prompts": { + "3": 0.013608596069092682, + "2": 0.014902802239145354, + "1": 0.015944909749231198 + }, + "rougeL__prompts": { + "3": 0.013608596069092682, + "2": 0.014902802239145354, + "1": 0.015944909749231198 + }, + "qf1__prompts": { + "3": 0.005311379630859115, + "2": 0.008663871869289476, + "1": 0.009659415077022301 + }, + "qem__prompts": { + "3": 0.011286681715575621, + "2": 0.013544018058690745, + "1": 0.011286681715575621 + } + }, + "e_tests_qa_AVERAGE|0": { + "rouge2": 0.003228194982976972, + "rouge2_stderr": 0.002547306056000719, + "rouge1": 0.014818769352489746, + "rouge1_stderr": 0.008024257279824093, + "rougeL": 0.014818769352489746, + "rougeL_stderr": 0.008024257279824093, + "qf1": 0.007878222192390297, + "qf1_stderr": 0.005529975861330393, + "qem": 0.012039127163280662, + "qem_stderr": 0.00874750521016363 + }, + "scio_SK_AVERAGE|0": { + "em_relaxed": 0.25434173669467786, + "acc": 0.250046685340803, + "em_relaxed_stderr": 0.02074943266589043, + "acc_stderr": 0.020633005498944187, + "em_relaxed__prompts": { + "3": 0.27675070028011206, + "2": 0.2549019607843137, + "1": 0.23137254901960785 + }, + "acc__prompts": { + "3": 0.2700280112044818, + "2": 0.24929971988795518, + "1": 0.230812324929972 + } + }, + "scio_AVERAGE|0": { + "em_relaxed": 0.25434173669467786, + "em_relaxed_stderr": 0.02074943266589043, + "acc": 0.250046685340803, + "acc_stderr": 0.020633005498944187 + }, + "arc_SK_AVERAGE|0": { + "em_relaxed": 0.24664955802680352, + "acc": 0.2446535500427716, + "em_relaxed_stderr": 0.012612937670503793, + "acc_stderr": 0.012578166445440875, + "em_relaxed__prompts": { + "2": 0.24721984602224123, + "3": 0.24550898203592814, + "1": 0.24721984602224123 + }, + "acc__prompts": { + "2": 0.24123182207014543, + "3": 0.24550898203592814, + "1": 0.24721984602224123 + } + }, + "arc_AVERAGE|0": { + "em_relaxed": 0.24664955802680352, + "em_relaxed_stderr": 0.012612937670503793, + "acc": 0.2446535500427716, + "acc_stderr": 0.012578166445440875 + }, + "sklep_rte_SK_AVERAGE|0": { + "em_norm_relaxed": 0.31506024096385543, + "acc": 0.3140562248995984, + "em_norm_relaxed_stderr": 0.010750305246379226, + "acc_stderr": 0.010735367420948657, + "em_norm_relaxed__prompts": { + "3": 0.16566265060240964, + "2": 0.5132530120481927, + "1": 0.26626506024096386 + }, + "acc__prompts": { + "3": 0.1650602409638554, + "2": 0.5132530120481927, + "1": 0.26385542168674697 + } + }, + "sklep_rte_AVERAGE|0": { + "em_norm_relaxed": 0.31506024096385543, + "em_norm_relaxed_stderr": 0.010750305246379226, + "acc": 0.3140562248995984, + "acc_stderr": 0.010735367420948657 + }, + "driving_test_SK_AVERAGE|0": { + "em_relaxed": 0.2710413694721826, + "acc": 0.2710413694721826, + "em_relaxed_stderr": 0.016450865339756114, + "acc_stderr": 0.016450865339756114, + "em_relaxed__prompts": { + "3": 0.17261055634807418, + "1": 0.3666191155492154, + "2": 0.2738944365192582 + }, + "acc__prompts": { + "3": 0.17261055634807418, + "1": 0.3666191155492154, + "2": 0.2738944365192582 + } + }, + "driving_test_AVERAGE|0": { + "em_relaxed": 0.2710413694721826, + "em_relaxed_stderr": 0.016450865339756114, + "acc": 0.2710413694721826, + "acc_stderr": 0.016450865339756114 + }, + "sklep_sentiment_analysis_SK_AVERAGE|0": { + "em_norm_relaxed": 0.11388355726167626, + "acc": 0.10748560460652591, + "em_norm_relaxed_stderr": 0.009453015726623323, + "acc_stderr": 0.009172256989852929, + "em_norm_relaxed__prompts": { + "2": 0.19193857965451055, + "3": 0.09404990403071017, + "1": 0.05566218809980806 + }, + "acc__prompts": { + "2": 0.1833013435700576, + "3": 0.09213051823416507, + "1": 0.04702495201535509 + } + }, + "sklep_sentiment_analysis_AVERAGE|0": { + "em_norm_relaxed": 0.11388355726167626, + "em_norm_relaxed_stderr": 0.009453015726623323, + "acc": 0.10748560460652591, + "acc_stderr": 0.009172256989852929 + }, + "slovakcopa_SK_AVERAGE|0": { + "em_relaxed": 0.27066666666666667, + "acc": 0.27, + "em_relaxed_stderr": 0.016480088564366616, + "acc_stderr": 0.016473550449007283, + "em_relaxed__prompts": { + "1": 0.402, + "2": 0.394, + "3": 0.016 + }, + "acc__prompts": { + "1": 0.402, + "2": 0.392, + "3": 0.016 + } + }, + "slovakcopa_AVERAGE|0": { + "em_relaxed": 0.27066666666666667, + "em_relaxed_stderr": 0.016480088564366616, + "acc": 0.27, + "acc_stderr": 0.016473550449007283 + }, + "sklep_qa_SK_AVERAGE|0": { + "qf1": 0.14283353603077906, + "qem": 0.05203659257713312, + "qf1_stderr": 0.0027203043184777205, + "qem_stderr": 0.002268897741053689, + "qf1__prompts": { + "2": 0.14428713100967, + "1": 0.1424834912487429, + "3": 0.1417299858339243 + }, + "qem__prompts": { + "2": 0.05238443076280914, + "1": 0.05248878221851195, + "3": 0.05123656475007826 + } + }, + "sklep_qa_AVERAGE|0": { + "qf1": 0.14283353603077906, + "qf1_stderr": 0.0027203043184777205, + "qem": 0.05203659257713312, + "qem_stderr": 0.002268897741053689 + }, + "senti_sk_3class_SK_AVERAGE|0": { + "em_norm_relaxed": 0.38621623547858314, + "acc": 0.3813698239612287, + "em_norm_relaxed_stderr": 0.006810719788072698, + "acc_stderr": 0.006801058501709849, + "em_norm_relaxed__prompts": { + "3": 0.5809279452640581, + "2": 0.32050459696386574, + "1": 0.2572161642078255 + }, + "acc__prompts": { + "3": 0.573444515715202, + "2": 0.3158007269617276, + "1": 0.25486422920675644 + } + }, + "senti_sk_3class_AVERAGE|0": { + "em_norm_relaxed": 0.38621623547858314, + "em_norm_relaxed_stderr": 0.006810719788072698, + "acc": 0.3813698239612287, + "acc_stderr": 0.006801058501709849 + }, + "multi_wiki_qa_SK_AVERAGE|0": { + "qf1": 0.12354806704071784, + "qem": 0.07884730838503101, + "qf1_stderr": 0.003950374051156623, + "qem_stderr": 0.0038122173887514235, + "qf1__prompts": { + "3": 0.12171966052606027, + "2": 0.12697709233547141, + "1": 0.12194744826062183 + }, + "qem__prompts": { + "3": 0.07884730838503101, + "2": 0.08204922953772263, + "1": 0.0756453872323394 + } + }, + "multi_wiki_qa_AVERAGE|0": { + "qf1": 0.12354806704071784, + "qf1_stderr": 0.003950374051156623, + "qem": 0.07884730838503101, + "qem_stderr": 0.0038122173887514235 + }, + "ifeval_sk_SK_AVERAGE|0": { + "prompt_level_strict_acc": 0.10242085661080075, + "prompt_level_loose_acc": 0.10242085661080075, + "inst_level_strict_acc": 0.21307506053268765, + "inst_level_loose_acc": 0.21307506053268765, + "prompt_level_strict_acc_stderr": 0.01309628698247887, + "prompt_level_loose_acc_stderr": 0.01309628698247887, + "inst_level_strict_acc_stderr": 0.00041218013349389904, + "inst_level_loose_acc_stderr": 0.00041218013349389904 + }, + "ifeval_sk_AVERAGE|0": { + "prompt_level_strict_acc": 0.10242085661080075, + "prompt_level_strict_acc_stderr": 0.01309628698247887, + "prompt_level_loose_acc": 0.10242085661080075, + "prompt_level_loose_acc_stderr": 0.01309628698247887, + "inst_level_strict_acc": 0.21307506053268765, + "inst_level_strict_acc_stderr": 0.00041218013349389904, + "inst_level_loose_acc": 0.21307506053268765, + "inst_level_loose_acc_stderr": 0.00041218013349389904 + }, + "globalmmlu_lite_SK_AVERAGE|0": { + "em_relaxed": 0.2783333333333333, + "acc": 0.2758333333333333, + "em_relaxed_stderr": 0.022429986391836288, + "acc_stderr": 0.022369608398602674, + "em_relaxed__prompts": { + "3": 0.2925, + "1": 0.2725, + "2": 0.27 + }, + "acc__prompts": { + "3": 0.2875, + "1": 0.2725, + "2": 0.2675 + } + }, + "globalmmlu_lite_AVERAGE|0": { + "em_relaxed": 0.2783333333333333, + "em_relaxed_stderr": 0.022429986391836288, + "acc": 0.2758333333333333, + "acc_stderr": 0.022369608398602674 + }, + "sklep_nli_SK_AVERAGE|0": { + "em_norm_relaxed": 0.024047428723687717, + "acc": 0.0187183586464162, + "em_norm_relaxed_stderr": 0.0017224579545798097, + "acc_stderr": 0.001563138066747669, + "em_norm_relaxed__prompts": { + "3": 0.002398081534772182, + "1": 0.004996003197442046, + "2": 0.06474820143884892 + }, + "acc__prompts": { + "3": 0.002398081534772182, + "1": 0.004396482813749001, + "2": 0.04936051159072742 + } + }, + "sklep_nli_AVERAGE|0": { + "em_norm_relaxed": 0.024047428723687717, + "em_norm_relaxed_stderr": 0.0017224579545798097, + "acc": 0.0187183586464162, + "acc_stderr": 0.001563138066747669 + }, + "slovak_tests_qa_SK_AVERAGE|0": { + "rouge2": 0.0030834556044640076, + "rouge1": 0.0149009568631566, + "rougeL": 0.0149009568631566, + "qf1": 0.008916490572837322, + "qem": 0.00980392156862745, + "rouge2_stderr": 0.00228223362995418, + "rouge1_stderr": 0.00523977237139308, + "rougeL_stderr": 0.00523977237139308, + "qf1_stderr": 0.005125765723300312, + "qem_stderr": 0.006367605535927983, + "rouge2__prompts": { + "1": 0.003274643190609577, + "2": 0.0021008403361344537, + "3": 0.0038748832866479918 + }, + "rouge1__prompts": { + "1": 0.014745243264740951, + "2": 0.015252537196116269, + "3": 0.014705090128612585 + }, + "rougeL__prompts": { + "1": 0.014745243264740951, + "2": 0.015252537196116269, + "3": 0.014705090128612585 + }, + "qf1__prompts": { + "1": 0.00765639589169001, + "2": 0.006349206349206349, + "3": 0.012743869477615606 + }, + "qem__prompts": { + "1": 0.008403361344537815, + "2": 0.008403361344537815, + "3": 0.012605042016806723 + } + }, + "slovak_tests_qa_AVERAGE|0": { + "rouge2": 0.0030834556044640076, + "rouge2_stderr": 0.00228223362995418, + "rouge1": 0.0149009568631566, + "rouge1_stderr": 0.00523977237139308, + "rougeL": 0.0149009568631566, + "rougeL_stderr": 0.00523977237139308, + "qf1": 0.008916490572837322, + "qf1_stderr": 0.005125765723300312, + "qem": 0.00980392156862745, + "qem_stderr": 0.006367605535927983 + }, + "skj1_SK_AVERAGE|0": { + "em_relaxed": 0.0033333333333333335, + "acc": 0.0, + "em_relaxed_stderr": 0.0033333333333333322, + "acc_stderr": 0.0, + "em_relaxed__prompts": { + "1": 0.01, + "3": 0.0, + "2": 0.0 + }, + "acc__prompts": { + "1": 0.0, + "3": 0.0, + "2": 0.0 + } + }, + "skj1_AVERAGE|0": { + "em_relaxed": 0.0033333333333333335, + "em_relaxed_stderr": 0.0033333333333333322, + "acc": 0.0, + "acc_stderr": 0.0 + }, + "financial_exams_SK_AVERAGE|0": { + "em_relaxed": 0.18640679660169915, + "acc": 0.18615692153923039, + "em_relaxed_stderr": 0.010665889604449459, + "acc_stderr": 0.010660482281487506, + "em_relaxed__prompts": { + "3": 0.18590704647676162, + "1": 0.18290854572713644, + "2": 0.1904047976011994 + }, + "acc__prompts": { + "3": 0.18590704647676162, + "1": 0.18290854572713644, + "2": 0.1896551724137931 + } + }, + "financial_exams_AVERAGE|0": { + "em_relaxed": 0.18640679660169915, + "em_relaxed_stderr": 0.010665889604449459, + "acc": 0.18615692153923039, + "acc_stderr": 0.010660482281487506 + }, + "truthful_qa_SK_AVERAGE|0": { + "em_relaxed": 0.32390745501285345, + "acc": 0.31876606683804626, + "em_relaxed_stderr": 0.01598799102785622, + "acc_stderr": 0.015861196644299858, + "em_relaxed__prompts": { + "2": 0.40359897172236503, + "3": 0.14524421593830333, + "1": 0.42287917737789205 + }, + "acc__prompts": { + "2": 0.397172236503856, + "3": 0.13624678663239073, + "1": 0.42287917737789205 + } + }, + "truthful_qa_AVERAGE|0": { + "em_relaxed": 0.32390745501285345, + "em_relaxed_stderr": 0.01598799102785622, + "acc": 0.31876606683804626, + "acc_stderr": 0.015861196644299858 + }, + "sklep_hate_speech_SK_AVERAGE|0": { + "em_norm_relaxed": 0.21430376547889815, + "acc": 0.2137983320697498, + "em_norm_relaxed_stderr": 0.010710417940514876, + "acc_stderr": 0.010698565134938456, + "em_norm_relaxed__prompts": { + "3": 0.09401061410159212, + "1": 0.35329795299469297, + "2": 0.1956027293404094 + }, + "acc__prompts": { + "3": 0.0932524639878696, + "1": 0.3525398028809704, + "2": 0.1956027293404094 + } + }, + "sklep_hate_speech_AVERAGE|0": { + "em_norm_relaxed": 0.21430376547889815, + "em_norm_relaxed_stderr": 0.010710417940514876, + "acc": 0.2137983320697498, + "acc_stderr": 0.010698565134938456 + }, + "toxic_sk_SK_AVERAGE|0": { + "em_norm_relaxed": 0.19977298524404086, + "acc": 0.19901626939084374, + "em_norm_relaxed_stderr": 0.011151493876554907, + "acc_stderr": 0.011139685234547891, + "em_norm_relaxed__prompts": { + "3": 0.007945516458569807, + "2": 0.23723041997729852, + "1": 0.35414301929625425 + }, + "acc__prompts": { + "3": 0.007945516458569807, + "2": 0.23609534619750283, + "1": 0.3530079455164586 + } + }, + "toxic_sk_AVERAGE|0": { + "em_norm_relaxed": 0.19977298524404086, + "em_norm_relaxed_stderr": 0.011151493876554907, + "acc": 0.19901626939084374, + "acc_stderr": 0.011139685234547891 + }, + "slohry_SK_AVERAGE|0": { + "em_norm_relaxed": 0.016983016983016984, + "acc": 0.016983016983016984, + "em_norm_relaxed_stderr": 0.004436902963904142, + "acc_stderr": 0.004436902963904142, + "em_norm_relaxed__prompts": { + "1": 0.018731268731268732, + "3": 0.016983016983016984, + "2": 0.015234765234765234 + }, + "acc__prompts": { + "1": 0.018731268731268732, + "3": 0.016983016983016984, + "2": 0.015234765234765234 + } + }, + "slohry_AVERAGE|0": { + "em_norm_relaxed": 0.016983016983016984, + "em_norm_relaxed_stderr": 0.004436902963904142, + "acc": 0.016983016983016984, + "acc_stderr": 0.004436902963904142 + }, + "chiby_SK_AVERAGE|0": { + "em_norm_relaxed": 0.14238539238539238, + "acc": 0.1411227661227661, + "em_norm_relaxed_stderr": 0.005425997886686588, + "acc_stderr": 0.005408227359113877, + "em_norm_relaxed__prompts": { + "2": 0.02564102564102564, + "3": 0.16258741258741258, + "1": 0.23892773892773891 + }, + "acc__prompts": { + "2": 0.025349650349650348, + "3": 0.1622960372960373, + "1": 0.23572261072261072 + } + }, + "chiby_AVERAGE|0": { + "em_norm_relaxed": 0.14238539238539238, + "em_norm_relaxed_stderr": 0.005425997886686588, + "acc": 0.1411227661227661, + "acc_stderr": 0.005408227359113877 + }, + "slovak_trivia_SK_AVERAGE|0": { + "em_relaxed": 0.2785646836638338, + "acc": 0.2785646836638338, + "em_relaxed_stderr": 0.023888375986348194, + "acc_stderr": 0.023888375986348194, + "em_relaxed__prompts": { + "3": 0.28611898016997167, + "1": 0.26628895184135976, + "2": 0.28328611898017 + }, + "acc__prompts": { + "3": 0.28611898016997167, + "1": 0.26628895184135976, + "2": 0.28328611898017 + } + }, + "slovak_trivia_AVERAGE|0": { + "em_relaxed": 0.2785646836638338, + "em_relaxed_stderr": 0.023888375986348194, + "acc": 0.2785646836638338, + "acc_stderr": 0.023888375986348194 + }, + "analogies_sk_SK_AVERAGE|0": { + "acc": 0.000628693574751666, + "acc_stderr": 0.00039549148554889166, + "acc__prompts": { + "3": 0.0, + "2": 0.0007544322897019992, + "1": 0.001131648434552999 + } + }, + "analogies_sk_AVERAGE|0": { + "acc": 0.000628693574751666, + "acc_stderr": 0.00039549148554889166 + }, + "SK_AVERAGE|0": { + "em_norm_relaxed": 0.20551192367489274, + "acc": 0.21129673365203147, + "em_relaxed": 0.24052159099864373, + "rouge2": 0.00315582529372049, + "rouge1": 0.014859863107823173, + "rougeL": 0.014859863107823173, + "qf1": 0.07079407895918113, + "qem": 0.038181737423518054, + "prompt_level_strict_acc": 0.10242085661080075, + "prompt_level_loose_acc": 0.10242085661080075, + "inst_level_strict_acc": 0.21307506053268765, + "inst_level_loose_acc": 0.21307506053268765, + "em_norm_relaxed_stderr": 0.0025722619842950684, + "acc_stderr": 0.0031822329617778393, + "em_relaxed_stderr": 0.0056971140284625835, + "rouge2_stderr": 0.001710070063230727, + "rouge1_stderr": 0.0047917616644827035, + "rougeL_stderr": 0.0047917616644827035, + "qf1_stderr": 0.002234105218066023, + "qem_stderr": 0.0029234626947495608, + "prompt_level_strict_acc_stderr": 0.01309628698247887, + "prompt_level_loose_acc_stderr": 0.01309628698247887, + "inst_level_strict_acc_stderr": 0.00041218013349389904, + "inst_level_loose_acc_stderr": 0.00041218013349389904 + }, + "AVERAGE|0": { + "em_norm_relaxed": 0.185433666807816, + "acc": 0.21317053562216026, + "em_relaxed": 0.2602872791374196, + "rouge2": 0.00315582529372049, + "rouge1": 0.014859863107823173, + "rougeL": 0.014859863107823173, + "qf1": 0.07079407895918113, + "qem": 0.038181737423518054, + "prompt_level_strict_acc": 0.10242085661080075, + "prompt_level_loose_acc": 0.10242085661080075, + "inst_level_strict_acc": 0.21307506053268765, + "inst_level_loose_acc": 0.21307506053268765, + "em_norm_relaxed_stderr": 0.002756064620435493, + "acc_stderr": 0.003457735075973522, + "em_relaxed_stderr": 0.0061656193838727944, + "rouge2_stderr": 0.001710070063230727, + "rouge1_stderr": 0.0047917616644827035, + "rougeL_stderr": 0.0047917616644827035, + "qf1_stderr": 0.002234105218066023, + "qem_stderr": 0.0029234626947495608, + "prompt_level_strict_acc_stderr": 0.01309628698247887, + "prompt_level_loose_acc_stderr": 0.01309628698247887, + "inst_level_strict_acc_stderr": 0.00041218013349389904, + "inst_level_loose_acc_stderr": 0.00041218013349389904 + } + } +} \ No newline at end of file diff --git a/perun/ministral_3b_aya/sk_bench/scores.csv b/perun/ministral_3b_aya/sk_bench/scores.csv new file mode 100644 index 0000000..7d6bd15 --- /dev/null +++ b/perun/ministral_3b_aya/sk_bench/scores.csv @@ -0,0 +1,30 @@ +group,task,metric,base_score,finetuned_score,difference_points +Knowledge & Reasoning,ARC,acc,24.4939,24.4654,-0.0285 +Knowledge & Reasoning,MMLU,acc,22.2452,24.1489,1.9037 +Knowledge & Reasoning,G-MMLU,acc,21.0833,27.5833,6.5000 +Knowledge & Reasoning,TruthQA,acc,6.4696,31.8766,25.4070 +Knowledge & Reasoning,SCIO,acc,23.8469,25.0047,1.1578 +Knowledge & Reasoning,FinExam,acc,8.8706,18.6157,9.7451 +Knowledge & Reasoning,Driving,acc,13.7423,27.1041,13.3618 +Knowledge & Reasoning,COPA,acc,5.3333,27.0000,21.6667 +Knowledge & Reasoning,SKJ1,acc,59.6667,0.0000,-59.6667 +Knowledge & Reasoning,Trivia,acc,19.9245,27.8565,7.9320 +Reading & QA,Belebele,acc,25.3704,26.6667,1.2963 +Reading & QA,Chiby,acc,0.0000,14.1123,14.1123 +Reading & QA,WikiQA,qem,0.0534,7.8847,7.8313 +Reading & QA,ETest-M,acc,15.0087,18.1501,3.1414 +Reading & QA,ETest-Q,qem,0.2257,1.2039,0.9782 +Reading & QA,SKTest-M,acc,17.3900,26.0080,8.6180 +Reading & QA,SKTest-Q,qem,1.2605,0.9804,-0.2801 +Reading & QA,sKLEP-QA,qem,0.0278,5.2037,5.1759 +Classification,sKLEP-NLI,acc,0.0000,1.8718,1.8718 +Classification,sKLEP-RTE,acc,0.0000,31.4056,31.4056 +Classification,Senti-3,acc,0.0000,38.1370,38.1370 +Classification,Senti-2,acc,0.0000,48.7393,48.7393 +Classification,sKLEP-Sent,acc,0.0000,10.7486,10.7486 +Classification,MovieRev,acc,0.0000,14.5750,14.5750 +Classification,Toxic,acc,0.0000,19.9016,19.9016 +Classification,sKLEP-Hate,acc,0.0000,21.3798,21.3798 +Instruction,IFEval,prompt_level_strict_acc,7.8212,10.2421,2.4209 +Grammar & Morphology,Analogies,acc,0.0000,0.0629,0.0629 +Grammar & Morphology,Slohry,acc,0.2331,1.6983,1.4652 diff --git a/perun/ministral_3b_aya/sk_bench/tasks_sk0_385.txt b/perun/ministral_3b_aya/sk_bench/tasks_sk0_385.txt new file mode 100644 index 0000000..323c524 --- /dev/null +++ b/perun/ministral_3b_aya/sk_bench/tasks_sk0_385.txt @@ -0,0 +1,385 @@ +arc.prompt3_mqa_single_correct_sk|0 +arc.prompt1_mqa_single_correct_sk|0 +arc.prompt2_mqa_single_correct_sk|0 +belebele.prompt3_mqa_single_correct_sk|0 +belebele.prompt1_mqa_single_correct_sk|0 +belebele.prompt2_mqa_single_correct_sk|0 +chiby.prompt2_chiby_sk|0 +chiby.prompt1_chiby_sk|0 +chiby.prompt3_chiby_sk|0 +financial_exams.prompt1_completion_sk|0 +financial_exams.prompt3_completion_sk|0 +financial_exams.prompt2_completion_sk|0 +ifeval_sk.ifeval|0 +mmlu.prompt3_mqa_single_correct_sk:abstract_algebra|0 +mmlu.prompt1_mqa_single_correct_sk:abstract_algebra|0 +mmlu.prompt2_mqa_single_correct_sk:abstract_algebra|0 +mmlu.prompt3_mqa_single_correct_sk:anatomy|0 +mmlu.prompt1_mqa_single_correct_sk:anatomy|0 +mmlu.prompt2_mqa_single_correct_sk:anatomy|0 +mmlu.prompt3_mqa_single_correct_sk:astronomy|0 +mmlu.prompt1_mqa_single_correct_sk:astronomy|0 +mmlu.prompt2_mqa_single_correct_sk:astronomy|0 +mmlu.prompt3_mqa_single_correct_sk:business_ethics|0 +mmlu.prompt1_mqa_single_correct_sk:business_ethics|0 +mmlu.prompt2_mqa_single_correct_sk:business_ethics|0 +mmlu.prompt3_mqa_single_correct_sk:clinical_knowledge|0 +mmlu.prompt1_mqa_single_correct_sk:clinical_knowledge|0 +mmlu.prompt2_mqa_single_correct_sk:clinical_knowledge|0 +mmlu.prompt3_mqa_single_correct_sk:college_biology|0 +mmlu.prompt1_mqa_single_correct_sk:college_biology|0 +mmlu.prompt2_mqa_single_correct_sk:college_biology|0 +mmlu.prompt3_mqa_single_correct_sk:college_chemistry|0 +mmlu.prompt1_mqa_single_correct_sk:college_chemistry|0 +mmlu.prompt2_mqa_single_correct_sk:college_chemistry|0 +mmlu.prompt3_mqa_single_correct_sk:college_computer_science|0 +mmlu.prompt1_mqa_single_correct_sk:college_computer_science|0 +mmlu.prompt2_mqa_single_correct_sk:college_computer_science|0 +mmlu.prompt3_mqa_single_correct_sk:college_mathematics|0 +mmlu.prompt1_mqa_single_correct_sk:college_mathematics|0 +mmlu.prompt2_mqa_single_correct_sk:college_mathematics|0 +mmlu.prompt3_mqa_single_correct_sk:college_medicine|0 +mmlu.prompt1_mqa_single_correct_sk:college_medicine|0 +mmlu.prompt2_mqa_single_correct_sk:college_medicine|0 +mmlu.prompt3_mqa_single_correct_sk:college_physics|0 +mmlu.prompt1_mqa_single_correct_sk:college_physics|0 +mmlu.prompt2_mqa_single_correct_sk:college_physics|0 +mmlu.prompt3_mqa_single_correct_sk:computer_security|0 +mmlu.prompt1_mqa_single_correct_sk:computer_security|0 +mmlu.prompt2_mqa_single_correct_sk:computer_security|0 +mmlu.prompt3_mqa_single_correct_sk:conceptual_physics|0 +mmlu.prompt1_mqa_single_correct_sk:conceptual_physics|0 +mmlu.prompt2_mqa_single_correct_sk:conceptual_physics|0 +mmlu.prompt3_mqa_single_correct_sk:econometrics|0 +mmlu.prompt1_mqa_single_correct_sk:econometrics|0 +mmlu.prompt2_mqa_single_correct_sk:econometrics|0 +mmlu.prompt3_mqa_single_correct_sk:electrical_engineering|0 +mmlu.prompt1_mqa_single_correct_sk:electrical_engineering|0 +mmlu.prompt2_mqa_single_correct_sk:electrical_engineering|0 +mmlu.prompt3_mqa_single_correct_sk:elementary_mathematics|0 +mmlu.prompt1_mqa_single_correct_sk:elementary_mathematics|0 +mmlu.prompt2_mqa_single_correct_sk:elementary_mathematics|0 +mmlu.prompt3_mqa_single_correct_sk:formal_logic|0 +mmlu.prompt1_mqa_single_correct_sk:formal_logic|0 +mmlu.prompt2_mqa_single_correct_sk:formal_logic|0 +mmlu.prompt3_mqa_single_correct_sk:global_facts|0 +mmlu.prompt1_mqa_single_correct_sk:global_facts|0 +mmlu.prompt2_mqa_single_correct_sk:global_facts|0 +mmlu.prompt3_mqa_single_correct_sk:high_school_biology|0 +mmlu.prompt1_mqa_single_correct_sk:high_school_biology|0 +mmlu.prompt2_mqa_single_correct_sk:high_school_biology|0 +mmlu.prompt3_mqa_single_correct_sk:high_school_chemistry|0 +mmlu.prompt1_mqa_single_correct_sk:high_school_chemistry|0 +mmlu.prompt2_mqa_single_correct_sk:high_school_chemistry|0 +mmlu.prompt3_mqa_single_correct_sk:high_school_computer_science|0 +mmlu.prompt1_mqa_single_correct_sk:high_school_computer_science|0 +mmlu.prompt2_mqa_single_correct_sk:high_school_computer_science|0 +mmlu.prompt3_mqa_single_correct_sk:high_school_european_history|0 +mmlu.prompt1_mqa_single_correct_sk:high_school_european_history|0 +mmlu.prompt2_mqa_single_correct_sk:high_school_european_history|0 +mmlu.prompt3_mqa_single_correct_sk:high_school_geography|0 +mmlu.prompt1_mqa_single_correct_sk:high_school_geography|0 +mmlu.prompt2_mqa_single_correct_sk:high_school_geography|0 +mmlu.prompt3_mqa_single_correct_sk:high_school_government_and_politics|0 +mmlu.prompt1_mqa_single_correct_sk:high_school_government_and_politics|0 +mmlu.prompt2_mqa_single_correct_sk:high_school_government_and_politics|0 +mmlu.prompt3_mqa_single_correct_sk:high_school_macroeconomics|0 +mmlu.prompt1_mqa_single_correct_sk:high_school_macroeconomics|0 +mmlu.prompt2_mqa_single_correct_sk:high_school_macroeconomics|0 +mmlu.prompt3_mqa_single_correct_sk:high_school_mathematics|0 +mmlu.prompt1_mqa_single_correct_sk:high_school_mathematics|0 +mmlu.prompt2_mqa_single_correct_sk:high_school_mathematics|0 +mmlu.prompt3_mqa_single_correct_sk:high_school_microeconomics|0 +mmlu.prompt1_mqa_single_correct_sk:high_school_microeconomics|0 +mmlu.prompt2_mqa_single_correct_sk:high_school_microeconomics|0 +mmlu.prompt3_mqa_single_correct_sk:high_school_physics|0 +mmlu.prompt1_mqa_single_correct_sk:high_school_physics|0 +mmlu.prompt2_mqa_single_correct_sk:high_school_physics|0 +mmlu.prompt3_mqa_single_correct_sk:high_school_psychology|0 +mmlu.prompt1_mqa_single_correct_sk:high_school_psychology|0 +mmlu.prompt2_mqa_single_correct_sk:high_school_psychology|0 +mmlu.prompt3_mqa_single_correct_sk:high_school_statistics|0 +mmlu.prompt1_mqa_single_correct_sk:high_school_statistics|0 +mmlu.prompt2_mqa_single_correct_sk:high_school_statistics|0 +mmlu.prompt3_mqa_single_correct_sk:high_school_us_history|0 +mmlu.prompt1_mqa_single_correct_sk:high_school_us_history|0 +mmlu.prompt2_mqa_single_correct_sk:high_school_us_history|0 +mmlu.prompt3_mqa_single_correct_sk:high_school_world_history|0 +mmlu.prompt1_mqa_single_correct_sk:high_school_world_history|0 +mmlu.prompt2_mqa_single_correct_sk:high_school_world_history|0 +mmlu.prompt3_mqa_single_correct_sk:human_aging|0 +mmlu.prompt1_mqa_single_correct_sk:human_aging|0 +mmlu.prompt2_mqa_single_correct_sk:human_aging|0 +mmlu.prompt3_mqa_single_correct_sk:human_sexuality|0 +mmlu.prompt1_mqa_single_correct_sk:human_sexuality|0 +mmlu.prompt2_mqa_single_correct_sk:human_sexuality|0 +mmlu.prompt3_mqa_single_correct_sk:international_law|0 +mmlu.prompt1_mqa_single_correct_sk:international_law|0 +mmlu.prompt2_mqa_single_correct_sk:international_law|0 +mmlu.prompt3_mqa_single_correct_sk:jurisprudence|0 +mmlu.prompt1_mqa_single_correct_sk:jurisprudence|0 +mmlu.prompt2_mqa_single_correct_sk:jurisprudence|0 +mmlu.prompt3_mqa_single_correct_sk:logical_fallacies|0 +mmlu.prompt1_mqa_single_correct_sk:logical_fallacies|0 +mmlu.prompt2_mqa_single_correct_sk:logical_fallacies|0 +mmlu.prompt3_mqa_single_correct_sk:machine_learning|0 +mmlu.prompt1_mqa_single_correct_sk:machine_learning|0 +mmlu.prompt2_mqa_single_correct_sk:machine_learning|0 +mmlu.prompt3_mqa_single_correct_sk:management|0 +mmlu.prompt1_mqa_single_correct_sk:management|0 +mmlu.prompt2_mqa_single_correct_sk:management|0 +mmlu.prompt3_mqa_single_correct_sk:marketing|0 +mmlu.prompt1_mqa_single_correct_sk:marketing|0 +mmlu.prompt2_mqa_single_correct_sk:marketing|0 +mmlu.prompt3_mqa_single_correct_sk:medical_genetics|0 +mmlu.prompt1_mqa_single_correct_sk:medical_genetics|0 +mmlu.prompt2_mqa_single_correct_sk:medical_genetics|0 +mmlu.prompt3_mqa_single_correct_sk:miscellaneous|0 +mmlu.prompt1_mqa_single_correct_sk:miscellaneous|0 +mmlu.prompt2_mqa_single_correct_sk:miscellaneous|0 +mmlu.prompt3_mqa_single_correct_sk:moral_disputes|0 +mmlu.prompt1_mqa_single_correct_sk:moral_disputes|0 +mmlu.prompt2_mqa_single_correct_sk:moral_disputes|0 +mmlu.prompt3_mqa_single_correct_sk:moral_scenarios|0 +mmlu.prompt1_mqa_single_correct_sk:moral_scenarios|0 +mmlu.prompt2_mqa_single_correct_sk:moral_scenarios|0 +mmlu.prompt3_mqa_single_correct_sk:nutrition|0 +mmlu.prompt1_mqa_single_correct_sk:nutrition|0 +mmlu.prompt2_mqa_single_correct_sk:nutrition|0 +mmlu.prompt3_mqa_single_correct_sk:philosophy|0 +mmlu.prompt1_mqa_single_correct_sk:philosophy|0 +mmlu.prompt2_mqa_single_correct_sk:philosophy|0 +mmlu.prompt3_mqa_single_correct_sk:prehistory|0 +mmlu.prompt1_mqa_single_correct_sk:prehistory|0 +mmlu.prompt2_mqa_single_correct_sk:prehistory|0 +mmlu.prompt3_mqa_single_correct_sk:professional_accounting|0 +mmlu.prompt1_mqa_single_correct_sk:professional_accounting|0 +mmlu.prompt2_mqa_single_correct_sk:professional_accounting|0 +mmlu.prompt3_mqa_single_correct_sk:professional_law|0 +mmlu.prompt1_mqa_single_correct_sk:professional_law|0 +mmlu.prompt2_mqa_single_correct_sk:professional_law|0 +mmlu.prompt3_mqa_single_correct_sk:professional_medicine|0 +mmlu.prompt1_mqa_single_correct_sk:professional_medicine|0 +mmlu.prompt2_mqa_single_correct_sk:professional_medicine|0 +mmlu.prompt3_mqa_single_correct_sk:professional_psychology|0 +mmlu.prompt1_mqa_single_correct_sk:professional_psychology|0 +mmlu.prompt2_mqa_single_correct_sk:professional_psychology|0 +mmlu.prompt3_mqa_single_correct_sk:public_relations|0 +mmlu.prompt1_mqa_single_correct_sk:public_relations|0 +mmlu.prompt2_mqa_single_correct_sk:public_relations|0 +mmlu.prompt3_mqa_single_correct_sk:security_studies|0 +mmlu.prompt1_mqa_single_correct_sk:security_studies|0 +mmlu.prompt2_mqa_single_correct_sk:security_studies|0 +mmlu.prompt3_mqa_single_correct_sk:sociology|0 +mmlu.prompt1_mqa_single_correct_sk:sociology|0 +mmlu.prompt2_mqa_single_correct_sk:sociology|0 +mmlu.prompt3_mqa_single_correct_sk:us_foreign_policy|0 +mmlu.prompt1_mqa_single_correct_sk:us_foreign_policy|0 +mmlu.prompt2_mqa_single_correct_sk:us_foreign_policy|0 +mmlu.prompt3_mqa_single_correct_sk:virology|0 +mmlu.prompt1_mqa_single_correct_sk:virology|0 +mmlu.prompt2_mqa_single_correct_sk:virology|0 +mmlu.prompt3_mqa_single_correct_sk:world_religions|0 +mmlu.prompt1_mqa_single_correct_sk:world_religions|0 +mmlu.prompt2_mqa_single_correct_sk:world_religions|0 +multi_wiki_qa.prompt3_qa_sk|0 +multi_wiki_qa.prompt2_qa_sk|0 +multi_wiki_qa.prompt1_qa_sk|0 +senti_sk.prompt2_sentiment_binary_sk|0 +senti_sk.prompt3_sentiment_binary_sk|0 +senti_sk.prompt1_sentiment_binary_sk|0 +senti_sk.prompt3_sentiment_3_class_sk|0 +senti_sk.prompt1_sentiment_3_class_sk|0 +senti_sk.prompt2_sentiment_3_class_sk|0 +skj1.prompt3_skj1_sk|0 +skj1.prompt1_skj1_sk|0 +skj1.prompt2_skj1_sk|0 +sklep.prompt2_sentiment_binary_sk:sentiment-analysis|0 +sklep.prompt3_sentiment_binary_sk:sentiment-analysis|0 +sklep.prompt1_sentiment_binary_sk:sentiment-analysis|0 +sklep.prompt1_hate_speech_sk:hate-speech|0 +sklep.prompt3_hate_speech_sk:hate-speech|0 +sklep.prompt2_hate_speech_sk:hate-speech|0 +sklep.prompt1_nli_3_classes_sk:nli|0 +sklep.prompt2_nli_3_classes_sk:nli|0 +sklep.prompt3_nli_3_classes_sk:nli|0 +sklep.prompt1_nli_binary_sk:rte|0 +sklep.prompt3_nli_binary_sk:rte|0 +sklep.prompt2_nli_binary_sk:rte|0 +sklep.prompt3_qa_sk:question-answering|0 +sklep.prompt2_qa_sk:question-answering|0 +sklep.prompt1_qa_sk:question-answering|0 +toxic_sk.prompt2_toxicity_sk|0 +toxic_sk.prompt3_toxicity_sk|0 +toxic_sk.prompt1_toxicity_sk|0 +truthfulqa.prompt3_mqa_single_correct_sk:mc1|0 +truthfulqa.prompt1_mqa_single_correct_sk:mc1|0 +truthfulqa.prompt2_mqa_single_correct_sk:mc1|0 +slovakcopa.prompt3_copa_sk|0 +slovakcopa.prompt1_copa_sk|0 +slovakcopa.prompt2_copa_sk|0 +movie_review.prompt3_sentiment_binary_sk|0 +movie_review.prompt2_sentiment_binary_sk|0 +movie_review.prompt1_sentiment_binary_sk|0 +slovak_trivia.prompt2_mqa_single_correct_sk|0 +slovak_trivia.prompt3_mqa_single_correct_sk|0 +slovak_trivia.prompt1_mqa_single_correct_sk|0 +analogies_sk.prompt1_analogy_sk|0 +analogies_sk.prompt2_analogy_sk|0 +analogies_sk.prompt3_analogy_sk|0 +driving_test.prompt3_mqa_single_correct_sk|0 +driving_test.prompt1_mqa_single_correct_sk|0 +driving_test.prompt2_mqa_single_correct_sk|0 +e_tests.prompt3_mqa_single_correct_sk:biologia|0 +e_tests.prompt3_mqa_single_correct_sk:obcianska-nauka|0 +e_tests.prompt3_mqa_single_correct_sk:informatika|0 +e_tests.prompt3_mqa_single_correct_sk:geografia|0 +e_tests.prompt3_mqa_single_correct_sk:matematika|0 +e_tests.prompt3_mqa_single_correct_sk:chemia|0 +e_tests.prompt3_mqa_single_correct_sk:dejepis|0 +e_tests.prompt3_mqa_single_correct_sk:fyzika|0 +e_tests.prompt3_mqa_single_correct_sk:slovensky-jazyk|0 +e_tests.prompt1_mqa_single_correct_sk:biologia|0 +e_tests.prompt1_mqa_single_correct_sk:obcianska-nauka|0 +e_tests.prompt1_mqa_single_correct_sk:informatika|0 +e_tests.prompt1_mqa_single_correct_sk:geografia|0 +e_tests.prompt1_mqa_single_correct_sk:matematika|0 +e_tests.prompt1_mqa_single_correct_sk:chemia|0 +e_tests.prompt1_mqa_single_correct_sk:dejepis|0 +e_tests.prompt1_mqa_single_correct_sk:fyzika|0 +e_tests.prompt1_mqa_single_correct_sk:slovensky-jazyk|0 +e_tests.prompt2_mqa_single_correct_sk:biologia|0 +e_tests.prompt2_mqa_single_correct_sk:obcianska-nauka|0 +e_tests.prompt2_mqa_single_correct_sk:informatika|0 +e_tests.prompt2_mqa_single_correct_sk:geografia|0 +e_tests.prompt2_mqa_single_correct_sk:matematika|0 +e_tests.prompt2_mqa_single_correct_sk:chemia|0 +e_tests.prompt2_mqa_single_correct_sk:dejepis|0 +e_tests.prompt2_mqa_single_correct_sk:fyzika|0 +e_tests.prompt2_mqa_single_correct_sk:slovensky-jazyk|0 +e_tests.prompt2_mqa_multiple_correct_sk:biologia|0 +e_tests.prompt2_mqa_multiple_correct_sk:obcianska-nauka|0 +e_tests.prompt2_mqa_multiple_correct_sk:informatika|0 +e_tests.prompt2_mqa_multiple_correct_sk:geografia|0 +e_tests.prompt2_mqa_multiple_correct_sk:matematika|0 +e_tests.prompt2_mqa_multiple_correct_sk:chemia|0 +e_tests.prompt2_mqa_multiple_correct_sk:dejepis|0 +e_tests.prompt2_mqa_multiple_correct_sk:fyzika|0 +e_tests.prompt2_mqa_multiple_correct_sk:slovensky-jazyk|0 +e_tests.prompt1_mqa_multiple_correct_sk:biologia|0 +e_tests.prompt1_mqa_multiple_correct_sk:obcianska-nauka|0 +e_tests.prompt1_mqa_multiple_correct_sk:informatika|0 +e_tests.prompt1_mqa_multiple_correct_sk:geografia|0 +e_tests.prompt1_mqa_multiple_correct_sk:matematika|0 +e_tests.prompt1_mqa_multiple_correct_sk:chemia|0 +e_tests.prompt1_mqa_multiple_correct_sk:dejepis|0 +e_tests.prompt1_mqa_multiple_correct_sk:fyzika|0 +e_tests.prompt1_mqa_multiple_correct_sk:slovensky-jazyk|0 +e_tests.prompt3_mqa_multiple_correct_sk:biologia|0 +e_tests.prompt3_mqa_multiple_correct_sk:obcianska-nauka|0 +e_tests.prompt3_mqa_multiple_correct_sk:informatika|0 +e_tests.prompt3_mqa_multiple_correct_sk:geografia|0 +e_tests.prompt3_mqa_multiple_correct_sk:matematika|0 +e_tests.prompt3_mqa_multiple_correct_sk:chemia|0 +e_tests.prompt3_mqa_multiple_correct_sk:dejepis|0 +e_tests.prompt3_mqa_multiple_correct_sk:fyzika|0 +e_tests.prompt3_mqa_multiple_correct_sk:slovensky-jazyk|0 +slovak_tests.prompt3_mqa_single_correct_sk:vyssie_studium|0 +slovak_tests.prompt3_mqa_single_correct_sk:monitor9|0 +slovak_tests.prompt3_mqa_single_correct_sk:maturita|0 +slovak_tests.prompt3_mqa_single_correct_sk:prijmacie_skusky|0 +slovak_tests.prompt1_mqa_single_correct_sk:vyssie_studium|0 +slovak_tests.prompt1_mqa_single_correct_sk:monitor9|0 +slovak_tests.prompt1_mqa_single_correct_sk:maturita|0 +slovak_tests.prompt1_mqa_single_correct_sk:prijmacie_skusky|0 +slovak_tests.prompt2_mqa_single_correct_sk:vyssie_studium|0 +slovak_tests.prompt2_mqa_single_correct_sk:monitor9|0 +slovak_tests.prompt2_mqa_single_correct_sk:maturita|0 +slovak_tests.prompt2_mqa_single_correct_sk:prijmacie_skusky|0 +e_tests.prompt1_qa_sk:biologia|0 +e_tests.prompt1_qa_sk:obcianska-nauka|0 +e_tests.prompt1_qa_sk:informatika|0 +e_tests.prompt1_qa_sk:geografia|0 +e_tests.prompt1_qa_sk:chemia|0 +e_tests.prompt1_qa_sk:dejepis|0 +e_tests.prompt1_qa_sk:fyzika|0 +e_tests.prompt1_qa_sk:slovensky-jazyk|0 +e_tests.prompt3_qa_sk:biologia|0 +e_tests.prompt3_qa_sk:obcianska-nauka|0 +e_tests.prompt3_qa_sk:informatika|0 +e_tests.prompt3_qa_sk:geografia|0 +e_tests.prompt3_qa_sk:chemia|0 +e_tests.prompt3_qa_sk:dejepis|0 +e_tests.prompt3_qa_sk:fyzika|0 +e_tests.prompt3_qa_sk:slovensky-jazyk|0 +e_tests.prompt2_qa_sk:biologia|0 +e_tests.prompt2_qa_sk:obcianska-nauka|0 +e_tests.prompt2_qa_sk:informatika|0 +e_tests.prompt2_qa_sk:geografia|0 +e_tests.prompt2_qa_sk:chemia|0 +e_tests.prompt2_qa_sk:dejepis|0 +e_tests.prompt2_qa_sk:fyzika|0 +e_tests.prompt2_qa_sk:slovensky-jazyk|0 +slovak_tests.prompt1_qa_sk:monitor9|0 +slovak_tests.prompt1_qa_sk:maturita|0 +slovak_tests.prompt1_qa_sk:prijmacie_skusky|0 +slovak_tests.prompt3_qa_sk:monitor9|0 +slovak_tests.prompt3_qa_sk:maturita|0 +slovak_tests.prompt3_qa_sk:prijmacie_skusky|0 +slovak_tests.prompt2_qa_sk:monitor9|0 +slovak_tests.prompt2_qa_sk:maturita|0 +slovak_tests.prompt2_qa_sk:prijmacie_skusky|0 +slohry.prompt2_declension_sk:declension_high|0 +slohry.prompt2_declension_sk:declension_mid|0 +slohry.prompt2_declension_sk:declension_low|0 +slohry.prompt1_declension_sk:declension_high|0 +slohry.prompt1_declension_sk:declension_mid|0 +slohry.prompt1_declension_sk:declension_low|0 +slohry.prompt3_declension_sk:declension_high|0 +slohry.prompt3_declension_sk:declension_mid|0 +slohry.prompt3_declension_sk:declension_low|0 +slohry.prompt3_gradation_sk:gradation_high|0 +slohry.prompt3_gradation_sk:gradation_mid|0 +slohry.prompt3_gradation_sk:gradation_low|0 +slohry.prompt1_gradation_sk:gradation_high|0 +slohry.prompt1_gradation_sk:gradation_mid|0 +slohry.prompt1_gradation_sk:gradation_low|0 +slohry.prompt2_gradation_sk:gradation_high|0 +slohry.prompt2_gradation_sk:gradation_mid|0 +slohry.prompt2_gradation_sk:gradation_low|0 +slohry.prompt1_identity_form_sk:identity_form_high|0 +slohry.prompt1_identity_form_sk:identity_form_mid|0 +slohry.prompt1_identity_form_sk:identity_form_low|0 +slohry.prompt2_identity_form_sk:identity_form_high|0 +slohry.prompt2_identity_form_sk:identity_form_mid|0 +slohry.prompt2_identity_form_sk:identity_form_low|0 +slohry.prompt3_identity_form_sk:identity_form_high|0 +slohry.prompt3_identity_form_sk:identity_form_mid|0 +slohry.prompt3_identity_form_sk:identity_form_low|0 +slohry.prompt2_inflection_sk:inflection_high|0 +slohry.prompt2_inflection_sk:inflection_mid|0 +slohry.prompt2_inflection_sk:inflection_low|0 +slohry.prompt1_inflection_sk:inflection_high|0 +slohry.prompt1_inflection_sk:inflection_mid|0 +slohry.prompt1_inflection_sk:inflection_low|0 +slohry.prompt3_inflection_sk:inflection_high|0 +slohry.prompt3_inflection_sk:inflection_mid|0 +slohry.prompt3_inflection_sk:inflection_low|0 +scio.prompt2_mqa_single_correct_sk:aj|0 +scio.prompt2_mqa_single_correct_sk:mat|0 +scio.prompt2_mqa_single_correct_sk:bio|0 +scio.prompt2_mqa_single_correct_sk:che|0 +scio.prompt2_mqa_single_correct_sk:vsp|0 +scio.prompt3_mqa_single_correct_sk:aj|0 +scio.prompt3_mqa_single_correct_sk:mat|0 +scio.prompt3_mqa_single_correct_sk:bio|0 +scio.prompt3_mqa_single_correct_sk:che|0 +scio.prompt3_mqa_single_correct_sk:vsp|0 +scio.prompt1_mqa_single_correct_sk:aj|0 +scio.prompt1_mqa_single_correct_sk:mat|0 +scio.prompt1_mqa_single_correct_sk:bio|0 +scio.prompt1_mqa_single_correct_sk:che|0 +scio.prompt1_mqa_single_correct_sk:vsp|0 +globalmmlu-lite.prompt3_mqa_single_correct_sk|0 +globalmmlu-lite.prompt1_mqa_single_correct_sk|0 +globalmmlu-lite.prompt2_mqa_single_correct_sk|0 diff --git a/perun/ministral_3b_aya/training/data_manifest.json b/perun/ministral_3b_aya/training/data_manifest.json new file mode 100644 index 0000000..227db5a --- /dev/null +++ b/perun/ministral_3b_aya/training/data_manifest.json @@ -0,0 +1,448 @@ +{ + "counts": { + "duplicate_pairs_removed": 101879, + "groups_with_multiple_responses": 133562, + "invalid_or_empty_rows": 283, + "prompt_groups": 2339573, + "retained_rows": 3492041, + "source_rows": 3594203 + }, + "files": { + "dataset_info.json": { + "bytes": 560, + "sha256": "049ff7f6702cdf4e49f0afb491bb2aad4552e96c96565f191564665fe8c74d1d" + }, + "eval.jsonl": { + "bytes": 475745, + "rows": 1000, + "sha256": "154ebcf5f73f1711865d389b3a7d483ca1006098ff5d25881bbabb5cfcb8a1e0" + }, + "eval.metadata.jsonl": { + "bytes": 416790, + "rows": 1000, + "sha256": "7864cb71daf35d56ea3caebb496108839d7d4e36653effd0741507e21a8e1e47" + }, + "test.jsonl": { + "bytes": 449384, + "rows": 1000, + "sha256": "9e6df865c825979b5b3d458a8a832406ce5a859e8f9aaa9e567b7377e41c1324" + }, + "test.metadata.jsonl": { + "bytes": 414881, + "rows": 1000, + "sha256": "a5a46723acc22f20c99a29e3b338d6986201c943c6b937f7402e2f04f671c8f3" + }, + "train.jsonl": { + "bytes": 1636857404, + "rows": 3490041, + "sha256": "39acf85df1401f9748f665d29754bd57234f45bca37195eb891f3ce696e5a1ab" + }, + "train.metadata.jsonl": { + "bytes": 1464274344, + "rows": 3490041, + "sha256": "2d8ce0a5394487463ea8292b6e9bf376a11dd4f79700292cf8a65fe9ca3a66a3" + } + }, + "format_version": 1, + "preparation_script_sha256": "5b126cdcfcaf0e38a38ff45f98fde9409b69eb10a925d6444990a90f99830b9b", + "prepared_at_utc": "2026-11-08T03:52:39.903856+00:00", + "requested_holdout_max_rows": { + "eval": 1000, + "test": 1000 + }, + "seed": 42, + "source": { + "bytes": 931479534, + "kind": "local_parquet", + "path": "/mnt/project/perun26012578/raw/slovak_train.parquet", + "revision": null, + "sha256": "6a21c52b80549cbb8f9d13de981a2d8f65c273a17f651fd09c1efcc952821e82" + }, + "source_breakdown": [ + { + "dataset_name": "Adversarial QA (T)", + "prompt_groups": 4, + "rows": 4, + "split": "eval", + "task_type": "question-answering" + }, + { + "dataset_name": "CNN-Daily-Mail (T)", + "prompt_groups": 29, + "rows": 29, + "split": "eval", + "task_type": "summarization" + }, + { + "dataset_name": "Dolly-v2 (T)", + "prompt_groups": 6, + "rows": 6, + "split": "eval", + "task_type": "generation" + }, + { + "dataset_name": "Flan-CoT-submix (T)", + "prompt_groups": 30, + "rows": 30, + "split": "eval", + "task_type": "generation" + }, + { + "dataset_name": "Flan-Coqa (T)", + "prompt_groups": 1, + "rows": 1, + "split": "eval", + "task_type": "question-answering" + }, + { + "dataset_name": "Flan-GEM-wiki-lingua (T)", + "prompt_groups": 9, + "rows": 9, + "split": "eval", + "task_type": "summarization" + }, + { + "dataset_name": "Flan-lambada (T)", + "prompt_groups": 2, + "rows": 2, + "split": "eval", + "task_type": "generation" + }, + { + "dataset_name": "Flan-unified-QA (T)", + "prompt_groups": 2, + "rows": 2, + "split": "eval", + "task_type": "question-answering" + }, + { + "dataset_name": "HotpotQA (T)", + "prompt_groups": 80, + "rows": 80, + "split": "eval", + "task_type": "question-answering" + }, + { + "dataset_name": "NQ-Open (T)", + "prompt_groups": 60, + "rows": 60, + "split": "eval", + "task_type": "question-answering" + }, + { + "dataset_name": "PAWS-Wiki (T)", + "prompt_groups": 7, + "rows": 7, + "split": "eval", + "task_type": "paraphrase-identification" + }, + { + "dataset_name": "PIQA (T)", + "prompt_groups": 7, + "rows": 7, + "split": "eval", + "task_type": "question-answering" + }, + { + "dataset_name": "SODA-inst (T)", + "prompt_groups": 34, + "rows": 267, + "split": "eval", + "task_type": "dialogue" + }, + { + "dataset_name": "UNER-LLM-inst", + "prompt_groups": 2, + "rows": 2, + "split": "eval", + "task_type": "named-entity-recognition" + }, + { + "dataset_name": "Wiki-split-inst (T)", + "prompt_groups": 329, + "rows": 334, + "split": "eval", + "task_type": "text-simplification" + }, + { + "dataset_name": "Xlel_wd-inst", + "prompt_groups": 4, + "rows": 4, + "split": "eval", + "task_type": "event-linking" + }, + { + "dataset_name": "Xlel_wd-inst (T)", + "prompt_groups": 156, + "rows": 156, + "split": "eval", + "task_type": "event-linking" + }, + { + "dataset_name": "Adversarial QA (T)", + "prompt_groups": 3, + "rows": 3, + "split": "test", + "task_type": "question-answering" + }, + { + "dataset_name": "CNN-Daily-Mail (T)", + "prompt_groups": 19, + "rows": 19, + "split": "test", + "task_type": "summarization" + }, + { + "dataset_name": "Dolly-v2 (T)", + "prompt_groups": 3, + "rows": 3, + "split": "test", + "task_type": "generation" + }, + { + "dataset_name": "Flan-CoT-submix (T)", + "prompt_groups": 28, + "rows": 28, + "split": "test", + "task_type": "generation" + }, + { + "dataset_name": "Flan-Coqa (T)", + "prompt_groups": 3, + "rows": 3, + "split": "test", + "task_type": "question-answering" + }, + { + "dataset_name": "Flan-GEM-wiki-lingua (T)", + "prompt_groups": 8, + "rows": 8, + "split": "test", + "task_type": "summarization" + }, + { + "dataset_name": "HotpotQA (T)", + "prompt_groups": 67, + "rows": 67, + "split": "test", + "task_type": "question-answering" + }, + { + "dataset_name": "Joke-explaination-inst (T)", + "prompt_groups": 1, + "rows": 1, + "split": "test", + "task_type": "generation" + }, + { + "dataset_name": "NQ-Open (T)", + "prompt_groups": 49, + "rows": 51, + "split": "test", + "task_type": "question-answering" + }, + { + "dataset_name": "PAWS-Wiki (T)", + "prompt_groups": 15, + "rows": 15, + "split": "test", + "task_type": "paraphrase-identification" + }, + { + "dataset_name": "PIQA (T)", + "prompt_groups": 7, + "rows": 7, + "split": "test", + "task_type": "question-answering" + }, + { + "dataset_name": "SODA-inst (T)", + "prompt_groups": 37, + "rows": 379, + "split": "test", + "task_type": "dialogue" + }, + { + "dataset_name": "UNER-LLM-inst", + "prompt_groups": 4, + "rows": 4, + "split": "test", + "task_type": "named-entity-recognition" + }, + { + "dataset_name": "Wiki-split-inst (T)", + "prompt_groups": 273, + "rows": 277, + "split": "test", + "task_type": "text-simplification" + }, + { + "dataset_name": "Xlel_wd-inst", + "prompt_groups": 4, + "rows": 4, + "split": "test", + "task_type": "event-linking" + }, + { + "dataset_name": "Xlel_wd-inst (T)", + "prompt_groups": 131, + "rows": 131, + "split": "test", + "task_type": "event-linking" + }, + { + "dataset_name": "Adversarial QA (T)", + "prompt_groups": 9686, + "rows": 9984, + "split": "train", + "task_type": "question-answering" + }, + { + "dataset_name": "CNN-Daily-Mail (T)", + "prompt_groups": 98366, + "rows": 98381, + "split": "train", + "task_type": "summarization" + }, + { + "dataset_name": "Dolly-v2 (T)", + "prompt_groups": 14559, + "rows": 14783, + "split": "train", + "task_type": "generation" + }, + { + "dataset_name": "Flan-CoT-submix (T)", + "prompt_groups": 88084, + "rows": 91852, + "split": "train", + "task_type": "generation" + }, + { + "dataset_name": "Flan-Coqa (T)", + "prompt_groups": 6405, + "rows": 6405, + "split": "train", + "task_type": "question-answering" + }, + { + "dataset_name": "Flan-GEM-wiki-lingua (T)", + "prompt_groups": 27128, + "rows": 27130, + "split": "train", + "task_type": "summarization" + }, + { + "dataset_name": "Flan-lambada (T)", + "prompt_groups": 4277, + "rows": 4277, + "split": "train", + "task_type": "generation" + }, + { + "dataset_name": "Flan-unified-QA (T)", + "prompt_groups": 538, + "rows": 538, + "split": "train", + "task_type": "question-answering" + }, + { + "dataset_name": "HotpotQA (T)", + "prompt_groups": 264664, + "rows": 266105, + "split": "train", + "task_type": "question-answering" + }, + { + "dataset_name": "Joke-explaination-inst (T)", + "prompt_groups": 724, + "rows": 733, + "split": "train", + "task_type": "generation" + }, + { + "dataset_name": "Mintaka-inst (T)", + "prompt_groups": 8, + "rows": 13834, + "split": "train", + "task_type": "question-answering" + }, + { + "dataset_name": "NQ-Open (T)", + "prompt_groups": 170153, + "rows": 172361, + "split": "train", + "task_type": "question-answering" + }, + { + "dataset_name": "PAWS-Wiki (T)", + "prompt_groups": 46682, + "rows": 46746, + "split": "train", + "task_type": "paraphrase-identification" + }, + { + "dataset_name": "PIQA (T)", + "prompt_groups": 16092, + "rows": 16093, + "split": "train", + "task_type": "question-answering" + }, + { + "dataset_name": "SODA-inst (T)", + "prompt_groups": 116604, + "rows": 1190936, + "split": "train", + "task_type": "dialogue" + }, + { + "dataset_name": "UNER-LLM-inst", + "prompt_groups": 8195, + "rows": 8197, + "split": "train", + "task_type": "named-entity-recognition" + }, + { + "dataset_name": "WIKI QA (T)", + "prompt_groups": 1030, + "rows": 1039, + "split": "train", + "task_type": "question-answering" + }, + { + "dataset_name": "Wiki-split-inst (T)", + "prompt_groups": 970135, + "rows": 988524, + "split": "train", + "task_type": "text-simplification" + }, + { + "dataset_name": "Xlel_wd-inst", + "prompt_groups": 12433, + "rows": 13268, + "split": "train", + "task_type": "event-linking" + }, + { + "dataset_name": "Xlel_wd-inst (T)", + "prompt_groups": 482400, + "rows": 518855, + "split": "train", + "task_type": "event-linking" + } + ], + "splits": { + "eval": { + "prompt_groups": 762, + "rows": 1000 + }, + "test": { + "prompt_groups": 652, + "rows": 1000 + }, + "train": { + "prompt_groups": 2338159, + "rows": 3490041 + } + } +} diff --git a/perun/ministral_3b_aya/training/eval_results.json b/perun/ministral_3b_aya/training/eval_results.json new file mode 100644 index 0000000..32563a7 --- /dev/null +++ b/perun/ministral_3b_aya/training/eval_results.json @@ -0,0 +1,7 @@ +{ + "epoch": 1.0, + "eval_loss": 1.1308903694152832, + "eval_runtime": 3.3257, + "eval_samples_per_second": 300.687, + "eval_steps_per_second": 9.622 +} \ No newline at end of file diff --git a/perun/ministral_3b_aya/training/run.json b/perun/ministral_3b_aya/training/run.json new file mode 100644 index 0000000..294ab0f --- /dev/null +++ b/perun/ministral_3b_aya/training/run.json @@ -0,0 +1,61 @@ +{ + "config": { + "model_name_or_path": "ministral/Ministral-3b-instruct", + "model_revision": "2c95908929198d6e69af8638f0dbbd9bc6b93f9e", + "stage": "sft", + "do_train": true, + "do_eval": true, + "finetuning_type": "lora", + "use_unsloth": true, + "use_unsloth_gc": true, + "bf16": true, + "infer_dtype": "bfloat16", + "flash_attn": "sdpa", + "dataset_dir": "/mnt/project/perun26012578/perun/data/aya_slovak_fixed", + "dataset": "aya_train", + "eval_dataset": "aya_eval", + "cutoff_len": 4096, + "packing": false, + "train_on_prompt": false, + "preprocessing_num_workers": 8, + "dataloader_num_workers": 2, + "overwrite_cache": false, + "lora_rank": 16, + "lora_alpha": 32, + "lora_dropout": 0.0, + "lora_target": "q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj", + "per_device_train_batch_size": 8, + "per_device_eval_batch_size": 8, + "gradient_accumulation_steps": 1, + "learning_rate": 0.0002, + "num_train_epochs": 1.0, + "lr_scheduler_type": "cosine", + "warmup_ratio": 0.08, + "optim": "adamw_torch", + "weight_decay": 0.0, + "max_grad_norm": 0.5, + "gradient_checkpointing": true, + "ddp_find_unused_parameters": false, + "output_dir": "/mnt/project/perun26012578/perun/models/ministral-3b-instruct/runs/full_aya_lr_0p000200_epoch1/adapter", + "overwrite_output_dir": false, + "logging_steps": 10, + "eval_strategy": "steps", + "eval_steps": 1000, + "save_strategy": "steps", + "save_steps": 1000, + "save_total_limit": 2, + "save_only_model": false, + "load_best_model_at_end": true, + "metric_for_best_model": "loss", + "greater_is_better": false, + "predict_with_generate": false, + "plot_loss": true, + "report_to": "none", + "seed": 42, + "data_seed": 42 + }, + "gpus": 4, + "effective_batch": 32, + "data_manifest_sha256": "ca76b5f54a7e899c535443db58d4438eb8267b974a99e928c9d83603b05afbc6", + "created_at_utc": "2026-11-09T16:46:52.760203+00:00" +} diff --git a/perun/ministral_3b_aya/training/train_results.json b/perun/ministral_3b_aya/training/train_results.json new file mode 100644 index 0000000..a392879 --- /dev/null +++ b/perun/ministral_3b_aya/training/train_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 1.0, + "total_flos": 3.332743094900831e+19, + "train_loss": 1.3334437627037912, + "train_runtime": 86760.7126, + "train_samples_per_second": 40.226, + "train_steps_per_second": 1.257 +} \ No newline at end of file diff --git a/perun/ministral_3b_aya/training/training_eval_loss.png b/perun/ministral_3b_aya/training/training_eval_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..65ba0480ae9ca53ad3862863054571b206112ab0 GIT binary patch literal 35015 zcmd?Rhd-8m_&$EwBU|<+luh;)p^Qp}>=nx1n>!M+l0*?oQpqg0y;r4-%HAVn?@izH z>iK*>zwiI>>-FmOJl*5E-q(8^=W!nAao#ak^|i=JnMn}@A-|-pVSpgG7zDwYASQ%Y zq{j2J zCT^SEd)KtKNFj9YlN6s{@D>5tuz52h0Dj#lrB$Sepn(xUWV)viG@t*H~*(hFSB5YV-5}6fxK=S+d zZ~xh64r0y&#nP?S*3E=M7Bv+;yK6hcZg4GjY}Q;?R^s(MeW`#<^fqUBc?Vxa{5{#N zm-U<>{$wIdLqp>}>{2epe14*KyZtNMxTUYJw6ya864BVb5Hfu5Vmf-Wr!TG0`|s0W z73@W)Y5CA2*}CH$Q&yEwRv55(m3QMvrB_l~nxoMQjB=p#cI*l`4KeH5K!k)n!!wHyNGUii81j{2$R8cZfAtTd3zj9}tfWVm^G zTe(P{YA7t2SI26DlAb+_;m`EPrRCFYK)3M?0dv+|BHEe;Sc1s4wKe+= z#iO<63wQ6nPv-W_Tr8~Cm-U#sKZ^aCcjc5#iE+i@O7$|9fsT%3a&odsol@!7gzU|m zH+@yY4wuVu5$oSyQ?M?9wHDesIdPuTGzUfPe@W`4{@yy1J+3{Z^dI zLuS7{yAVaeHpHV5<+-)*(j}`{U}4F;N~vSvY6w;-D$V5LzcEgj>M%UXGHbdHR{Zu z6}b`bpL%JE0TP)ZJA;J6a(deroytsvZk6JfkNUE>)~suylS^)r`r)_o(VF^5=;3mj zBg8(+*nq$rGB7gK;oqs_Sob z8)5UVAMcLKJU?50#&A)bK&`AbzwbX$BpdpYkoCGrS1Eqe0MP1{VV;(Qo-M^~p_M&zcEQBF(l%x_KN z(Gm;Xa%^dDpPr23c%XfzDQLAeYIiJ%l118C70xk^@U#6jRBFe(K~E-Wk*yH8&5Uid`<%g_YqxDI$_4f7!zxOEmF z=s`P2*of~@EbZgwC`ukd!H&Wy(^b)ciF-s;r%&I%_0isXuJZ-J1)=ZqP^8S5e@<_S zN#g!Q1m%TX1M8trcLuO7EQr;3T}ZrP(}xd25K3?~b4t{lW9fy%Vek-wian+^VP5cH zYlj^Uk28m48A9NZ^;=;jq-2K^`u(|7#lrjiM@dQd&>bo#Cl~$jVbhay*TwGqI8$93 zw6mlJm*=91S;FoDZX?A(yRP1Ue&jYfS?W7Yl?5)m%d?VXriRg~MG#IF!s%TK;ebIm zyVV~*05pDSTJGxG^*vLGMbN0QJtDJo%^Ix!3@&}AFEM-2evbxlja9M z@}EFt%&Dtmhdmj!ydPV(WhQOkbf4t&*RM^lF2+)Ea3p?o=$bk?Tx~~@1pu~laxSN2 zwXyH0ka_s8=jtEN@x<@Fm0ao%IOXM;F}#i6zkhE%T&+(;H#%V0-R)W-t{S`D5CSIv+_sR8m+dH)Fq24~nl zlxecT#3bv^xl%dD9(=Mt=RcFAwX_ok^yLxks863h(Q_0DiyVhETZs)k$Ae%f_}NZk z5=LC^0+aXxQLQ*w$d8QeSlAAH8TR+&g0u67tLEk|!-c(fBO$uI1oZqaebFN9c>BVQ z8#f|WunH>`>e2D>bl&nWIEll>?7uN`NlM;0d+uC`ppmJmlb~1So7K6jFpKE8xYN3P z$u7g+o}DLncSE2C_OS15%i#5kI2c&(6~lIU2+3*y6GgQ<&qo7j@YNW_ZSIR%HDCx! zYS)@h;Ud*uj!K*T>fAPxp)afIz-)qX|rrLM4(pYu+Zg{LZ{)zo7N_qBjF@o z+1sDXicGZ#yhKb%$p9qiK7>e_pl|3a2xl;SGG4jfciH46(1i;oCeOv(CsZ59$1T%T z!s>uLvOL+|Ir#bOF!_sJ6RfV=u)P@L{j@YjE+r*%e~6_n!||FqR16IKx>?GjD;2`W zq5o#R^M%5x?d@%>%f*WqFE~0nMu<#2&Fs*r_bM$d#YdV-Z+9&1oKOEAZc^a;_5XsP zV91s4Au7^FbIdtCj`%5;d25a~I=^007Q4&%dxi3Y=~gwxiQ-)!N`Dv9WWb|DFCj!3@5P4aU)a^1S<_W zig=&rANGxx8w=EmQON==nr=^Qe@G|DY?pA}p|hp+c5G~H2^`>Wa2#=PaInh)DWra* zJ`5D>iqUX7q@PDVeez@iAgUF10yR6k>+T4f-^e{+)JtQvEb!>Y^`S~DRf{4Tnwmo? zo12>vW+L^vbxtVz(@hh@u_-xDW60?lm?-K5zCwvO&t>P9m@ttMetV(NzvFeC$3wK; zQF3afdKEh?ZPP+LGc%L@=FOWg$4^DfAEMN@V!e$;JZQ(cdJdrTs)@-%z&tz@$6f38 z?a_D+LAD+EnNtu)-?dbRzjdM^DuJ~p566t0rd~k&mv$SA1GbGM^IvJJKiL*t8Y(*l zaHof*Hx7XlM@&xMDqKGDW1=!Hj!bdmGY|R#phK}^4>vqMxE{0>}?k`-!3S;G>?vbpDMRLmyCugm(5IvP8uV;)C7QL^!YP`9?*q?jHbuOf~}J*>)kP zWST4IWes8Tp7qlY3Z@v<(^2Nx9S>8HkdQzVm|-lX@*ZMSGO(hMo7?Ub8yhxGUbyz< z%NOI?K&h?3TJzS+`ubQwhuYn2U?}7k*Cb0UH(mN zZQ={+>P4~^`JX>)>(DOqmf$=+HQCB=GaqONHJoWtpse8l@P%r-E?2Ko0UPju%x2If z3nJfB$O2VP_V57e%R{tT(!;(VWSVXg3_R&*Wnp9-u*!m?X$wFP^p+#rFrUSRX1pfg zd}oGS1TdGkJ=8V;LX?+vbdr*itPjmcgEzlleRl(;y!Rm3Prc7C#GN?a&a2$*9}tHG zf(l5Wp_h=G9r7&q)MsMPX#_L#LU*8UvZ> zXkmKawYeNQqm5f`--D_$-Yks3wpyD_cbj?{#omO5gh!940P02{YxvIhrmyFaq54x$ zp=Et2HVlxSg-0u)1wCpbBWBfI-a5**!U459WiuHH2S?Z&hKBS=yE)N|{TDykv(){5 z{OJ1p>)mx~TG|MdmBOaEsHaC>cXHqq$EHBfrr_(my*@NGHSQOIUtCqiZ1C<=qb1JC z$%(Iu>8h}!(AmZKJUw!QC^Q%IVfye_Y*JmF zmMd$1h3t+-DXiAenqr+B`wC1;Y+6ZcAgkGf<#P6EMVF@l4&RQFxxTHBeu3ficbV}P zV6`I?7S%dLq!xF9CK*GV9mVRH7K-rE$BJ7w;X4nO5&m^>y z%_sEU)37%Mt3R&+Mu(voJ-}Xqjp_^5w3A>$13ZfLpS!x+ff}Mgx_@KnX_uLk+0h|Z z2Z5vkCZ@Z{vYxO&JawV3faQh&9k9W^iMBSEVXWHAtCIV-Gcq$1V9z{x`qTynrdOxw z?=O4d!i6;e0BJcD7Av_*zT|g3bB2XxF&@)xQqSWSyyaPC-I0*Poidl>`IJMAC^;tD`-St);=A z*ke!xkfr3R_38F$m?VumcixSr{yErQoLpTUpSk~KnaSnYsdM1F< zwM;ZdBxPofO$3w}lcikloo9$B8=jx?{|{D&UgeHcg7g4kN5OBU1xDd!t{4HMV=C^&M|n`5<#gP z4Y?ST%JEjR%b{rfQ8-8F9u7n!x6;K6BB|5BN!tC!L!#h7cCQpgl$J^;0`7>Ve}DAF z&i~*~-a3H9WI_2@>Egyd+|h+$y`@FtZSMl%ZpaNIBU>Q05Tou#Moy~y-A`N=&4@`mc=)gu8UNI;Ps7-8O*I9_Rk0qy0^4 zI=V;*8Q-LPFL`)~XNH{woH=vG3bF-k>0bao&eV=|I1)o+##^d4_-|KY=Z z$W-DFd}sFE??0{K&;GO8K6{3bk6S>1ROM(bVmWji74l#bT`prSf|kn&lUx4675CtB zVxdR=uV23&Q0jl&m6jvBafB=US6AUcexPs9r1@Hwx|X?R`p=;*ZD8$a7E=SH0@ch` zDyAu_hnHBlpE)J$MKV~R`CQMt>->txTR;7QH=-v3>o|u#6{PS~Z?0q|xQuC1f;I@9xaYZSZ^&eGc<>a1LbPqO=vMz^<Ng(0x97h#--DN<8DIK6Yzt<)$x+eaGu!zhw$f|v)s-ye`5(EbNKc(&VSC-p4D?sa+??&3{rNY# z&usj7wjeeve|-RZ2{IQ|S630>$S4Vh8Efe3qFn4N_^30GoSa;8w^sm77+=WyGy%Mc zdCq8$cy%eEn&eVK(E7-A(b_EiDh-9UX*{Rv4Mb!_V%NVFa^pQ%A4MN=b22|MD<(1SnHdKamws4$R*a!EkOcIoMeqof$bGzFRn#&xbEvfvSk-Rd z=7q)7jAOh9QZuKFOnJ}t8HexBBlXgyxl~j*e0+RlgZIQ{Jmm_j7T)t|v6+kX0S9?=;9sL;5i)x%WiP}A4LF}EJMzV5Z!ZZ`R}GP6#OXAfcA(>-C1usUB=DTv*1Rh zSpux9%)v+uEkDY-D=I1o!+b2iLHhYQiRbORcWV8G<}RBe=HINuB&+dz;6;}LpMrQJ zDttQ!2Od#T%9k%+&O^-25f1As4LjNA2P);fNhht9-#fK5_z}GVoP>y&mkym7e|rDX ztzY5^94ZML8yke>i1Ffe+wEn~aY{pwA0cDE0Fuzu^mI~kas&|d>Vuy@yaE<}odb?y z-4adhd)V9CI|;kI`;WKw<;$htD+wh39=axO_899H?0>NQ5>$G3>F^p~vN^UBJ~_xY zxYC=?0CCxOX#giIEDXfs{K1On+{EDp>_miws74*{yKT;jW?!jy7v;(`@w8r8X5Y@b zlfP3(Q8o44r_yU^WUjcPf)SRd5rc!rTQyuQ>FrASU7Kful71^@T*}I0Orp1KA=8A2UcDbLGJCFrEXm0~2*u1zDMgN%co0T| zOlk^lzi3~ZPE_&rIUCMNd|_Je+_E+mKC7H(%b(aQi`#*qVp974B&RBSQr^$u78X{} zx~Vqy`R77eUh6P(IOjx8qB5HM=aiRMgsamwK%AmMPx04|j`a@FdfAO!W+1;V%s!Xu zz7H=ZqMG*9KBwEvo$VpEsZo&2=|wF@oS&f*EaZ#M>*B!4U7g44*gzHnb$pPH@|E-I zfJV4`sCE)whEfnc1SE*lLHV-*UJss*9IF00-N(Vry?Xt|4ZC#a0?^dY!2;b~7x#mi zk7&efZEX?eGgeASNAY3MYS&EYNKldzTG+bl_dkWMxMnS!pz|QOLJqF<_0FMs$_GAWe3M0MG;w2~`{6 zQ1eCI8OZP`ET4jU6Fn5AnWF|PoFn*fkF-b|IWHlB7HVu8FgS?Nb#F3uMaCBjer-I7 zia(a>Dl04f=U+y-19(htcB#1aE}B$_M1+T@A@C>xjFxpF9fI<8=-ira5Fvm5{9#lM zmc_b`4lRWO1e~}3)_{iJPoE^UqAsDcL|?;=Q}e{4s2vVz(&NWJoGTqN0|?>yDj;_< z^)`EYclmnqE z0c0rHFP?$5>`>yu4jUR9M*}{QF-sCBDj%E$DLHts|EHrQF$Ki~lY&`J7$Rj(|Kc_H z*wBy(T$&==;2jkU%c)AUu?abg`r5NNkk84iuZI2oCockhWLC5^H;)G(oP^ZwuJ^z$ zny~FnCzju3W;BEu>avqghg1+JMk5@O(a*m97ZVfG2}c@-5EQ1C$HP%Y|Le|Muw{!IBa9f_5`J_VZZLYi*{Dl~Q(HozW?5Xbm& zBdCv(&;GQxwpv5L(~n7wjvk3962Y52wO8$u+3$^0jhp@`CR*`&NQe>$PmeM(zV&g5 z_KvKk^}C6_AbbEDbqVr6GFYIuZ{NDlD;Fw?l(GIR9dlTTFu0Z|&$wl2S)u};Yg~V+dcFg=}wG8C&%U=_irAX0Z0Wxm%?~YU+K0ZRI z%OTxX9E&A?_ogZ1I8@jHBTxeAKGi}pEUu%YgQm4`K2gdF6*xehXUUxuWlxOaaCfA` zGKCtGaHUS%j6~k-uFfyGLuLFy&&-j?rFXA?yNE+U40K6+*NRC(P`(M_7^W%siAXY& zpKNCRMT&Jjt^;WegMawL$xM3s^ugMiI}}4l3Wv)+d?14uippN=pd(Dpd`YgD%M7|| zXlSoT z>&azawAibiYw$WRZyM&U0cy&TVSf+Di7D#bYmH0oNI-HJoGC$zIH1DWe0j`E727&L zR3->wOR-gBct_j%dOJrBupk7(VBwDgy9-tO<-O;T5@YA`jJr^5@|bAA30`ky&_zax zt1+WjgWqxldKU*rtBgMWJ>>}Y19PFve9| zfZo>eGCsvze#t_&oKTerc}Zg^l!&80=U?Kr(XEyr<1HFC2&}DZ{L0%`=DHnNJG#^H z-R*7v;^dq|n9CjK(UCUoGmi;}Kl~&OgH`lL33($C$Ha4|LN$<%#scQp(0aYFytEvV zDKh%#2KR;Fx1{E;@T??H?TurdjiLZhXuQ(OlVIMyBFjV)T0Sd~15w5e6b33{gR|2Hhc>wL5A5dZsWg4cB=f^Q3NrL= z6B<1Luy&mQetJ<;lW;6}QxEVEg47hON4rb;=!5zBc|*2EkZK!2oL>9FpVifW-Bwpy8aj8(@wK=s9GSXtjwWcug%oG3D{&aI~oN_n^O?3V3j11d;96C+858F}jg7 zf>ZR_z}KS_6SY)T5i|mT%rN~hQ7fUb3^Uwt&57jwBAKyQ1(oMJM#5yKD<5RTDY!xh zj^;lb`Y}*ZUsE3|rgy)2lMX77*;mglVOhuspQNS^$F~q5IX%&k0BdMzsh^=4#HQ== zIhL8=6A)O#^Q0*SNyC9WLvtx2gjoU0NqkmBL{mfK{^-P$sw#PyFMNFbY%G=@NG_Hn zH~nF*l85lbta!S+x;%;Eg7a27LZ#^jOzGcyBYZ4 z8mxJxypSg@tkmZ~N=}4P*IxI}Fb-0zlJ_`5Ejs$`sO}4Sx?8typ-}7NBZby#vzBSL zrQifUG8PKoHdM*ErFtQa^wOjN<9A$NyQkICvJjr-`wC4q*x*A?%2)=fM1B|l(ncDd z#+CBza-pP&1nze^FqAX8(T?f0QcbKBj*!nSU6eAgr6E=47VWwLac5eG};j6Azs}ycd6NqW8@wvPr=JlG zig%Np1nQSLMK9ElUsQKj;mx3PTUozFENX>%_wF6wJeoX%O=y@t^_oD+e18!KVg=SE zMmf(jRer0k!(mWwJu4`v4o^~S_l0J}ci06xhp*Ov&I5!@ic6O+p|}s4VH{xpfMFYy z$>A$#h7Q~{9E9$P2Q04c-!ARY;hLj{NcEtzSCO_I6 zxNH4=(rfvoN<*VAyRc9niVzi7jf@B z0!`+Rt06;*cZDY|J_c8meH^3z|QmPzRYah)p>cZ$I~1F6$pUdGYiqAzHUl zH8y7K`)~{I-HnH^K#bv>1L)=ftqe+v0%!qE+_sJL%$XM7j5|NyOrjd?mpeI{VmRK{ zC5BBJE=0^d@QZ117%pAwze9n~#l6A%_h?Q9RN+L>{20)RH;5szkY?BdeZxvz&i%3D zw!ZK5C3jeaJ;fBn1)xjS{V~zSBd#|L41P>We7_^a=R^jR3(80{5Xt2oCLnQvyDKh1 z$3O(D=bD?EL=LNdN&XKknUk*A948BRG|}DY<*F~GUwVzF5Qo3>GyM z03LmB^9K~Dc7eRtXe-dJQH|4e;R1tuGrekx*+X`=a2%xiqd*2T19eh4O^n9XFS+_* zpZGjXDm`;u#uPwehy&tVR#xWnJn(m)saq>}iz30!fE1H`^#?tu2fQK>rQfyQ{vhm0 zM^{iSGV0OJhU!RY5vO_?>L?dLP(Z98SfeI33Jzs5M2tD!AKYd|YizU)^s42z7ja!* z6Hp<9XQC#pEa-e4ZdzR)_jh;z!Bpfi*a=oft9g`^*ig|8!Fj7>&_NLe*8s zxEO&Q)C8L&DK#|;WY!Pn)qPJhu&;Fu3}`^&j{>V+4wMEMW!- z@!8qgzDmA6wK(|{bQc7ut%rI^$TP>?Uom>s55(#xo$Pr{e)bJqCX_*LqmNWeP z!=qWVB#=y$c+Kh0d&`6Us=bfE)W86B7fylg4x?f7#}JB+Evxl^UHfaFr>DmXn_^MM z=G(3WnJelVYiLkK87Y*&$Sn>%Z+pwLq1*h`D{j!6UxBrP+~&t0Z`A$Ar}x~>kHaz` zKn?;02nv||!YWcoP{=L9c2`ER0SGFa2Jd_7rb`#};8#?Rm+%ygXNHqlvB;L<_Oj?2 zmyAnXcGCQNpU@nC!2~a`evJqpXJB(xJeur=q%Q=lyZ8C|F~1~qk@WT;2TE zmf8760j3|X@Nkfz(f|%lh|g%!CUe`Il)^DsfU&`ogR|itCxpvi@<0WQy{zeR+vsXfHCqU91ohL$3qAVHv*KZ#TrQ-Mf57YfqgD6C&qK zaEAM?a6-hE#nGsAA`qQwvs0YN)|Ju`qKeqj1`Q2izT__m0{d4a;XZ5YL3P-4;lP;hXE0x^_gvlBcTkJ+U2#aHLDU+S#|oq@JWQ3~+2tn2z3b$34VDOf{V2i? zy~RgiqJ9kqF{?}9;C8QL_6ejtn)xsfx3V_B)bt$xu^*JEChu~6x5AxovzyBczXRzy z5>2_S?p#b0xe&@`A}mam?j`wp91950%z(~W8;CsL)mm6iTO4|+{yu|Iihg)i>P>Vm!!z})E;P| zpnS5tHqeQ9qo1c97$}eW4Qn<#6_&AGxvaHMCnk>sScSW@b-a;6eNv}VGffYvwz_SG zClcZmD(Bff-xB6I5xfens27@L#qQg4xVUIr3wLM+Ed z2K&K0p`RB6feX!E1GnbyflH&veD@aA!obey0v_4>JEYaue$dC^VypPRlnRYxE?tgl zgVAYZc7f;yB8+Z^EH%hCPJJKlnTf!?P!HwX-n+vf0v9PYyJhp{UcRgE6_Yp4)gdi%@Ncjz)IwNl@F39U^1%u`uefE9#X@XIfWf@ zp6ImBzo5d!pgURPmtv74j#;r44l08|4UP?Rt;a4wfOGUh#s~s*yah_e5r2N>@ra9; zTs*9Jt~;_3gy3X{88G^07Bq}4k&Y;5=kP#)1}$oqjs&%&MMIhnLme7GS>Zcaf;8GHCKINJ-Ge%wTJkiY{aAr)6&w~_LufQ4l(=w9C`i?Gbp4Q z9C}sz`hhpuJ!AYCqrR6{&7%gac5n41z!Q%|8I9o07kn^pQ0*WL#}~Lg&+hQ7Cs=e9 z54r+g@j)=QwJ9O))2x3=hL3~Er>*>V+NI#6r6*%;1Q?O|EN4Ar5LX+<5D!#;*14mx ztb(Asl>74+ZgR4Z@Kz?IIqAs{P5W^f>0WEtRFH-vOC8pNC;~~f;CFv7vzKrb zTr71D`VW<)!r|l9-2@{W+aX1lO1<$`zu27m-w&J5iZ>AIqFY0 z`_nk_&*gS6=*jx(Pe=bdBkJgv`mQxRpx4KD;iZmsF2GcmejQd4fp2b@0j8QcrG{4z zU}oGQ*BK*1XN|!cX6^OdDvtAaNioAd`i9Y4bYf+t1W7Qo8uXd?6;wzoq-fPlbK42H zRJkJGj72A3{BP(&0LXHG=n`kHOIxlQboy!3N0Ql;|$7X@I{L4EY9Y7KPcGxLL7_?MqC(wQhpT)sP`poorK z6-J)n!^}AGwd&sQAF(TVb}}gF-y_K$rj60SZexl*u~dXTz73P=Ajc&MZ;9h<5gS#p zX7pn8`{>mE?_IWtU7nFKBZ2()RYmA>KAA*)kZ=h?nS=&=G)Xp*V~1TglKtdTw*nVF z@^1%n^I_Ou3MZ?Xk}ug+Qo3vL{d;x8M;B4TY_m5AzJRL{`eDythR=8W+d4^!unBSO zrm%X1w1wgKzHL zG3{T@j;*=}W|7Z+@DPZBDZ^SyIfC0T(MgS9UO03VIo^^(=Y#?-)(|N&f#r2knhK|S z`{=@Tx0ZFU=R3sl=x(H`=}#r67!kr3 ztAt~1hiL0fFg<*u+d>$Fh@EMbpt8GSNU+t0<$nT`GZT=81R~E|geS24&gQRhVQ*`_ zXuPk!f73;o9VR<~9i~ZDr0nSVP)@795WRJ_k9^wse>-90B05z*>&|%zR?ka340;pk~$4z zOqv6PDlROfSgqNl(2Qq}1ZcAXz#0IX`t9A0&Q9^r#Mrn)d;IyJwUk@}PMn#HYHy zCqXDX2Un&tDtPj1j(IMR;xbZas}}89Ar1CeTy)r{-{s>uQo7Y{(}RU>c`h$AkaV%X zs2vLI3E-c^_Jp8nBUognAeRlVuU9$P`V)a_|4`j;fTSpcE->~Me}rpSuY#s`Ov_}j zl=KUc)&e8`Ps1MQR;q=#GST`aE2Ys2qs!;yne{zg1}!zO?e0R&8;p*RfKEbzLhJq@ zAC|tscLA3WlXT~)>x6Kgj8*X!9ZdngMEk0y!9Uu<~`x%*}z+-R5cv)@1}p zZ+6fBhU8nUJ-7o3T&EjDB^(#x&FBH_KS%kS<5qn>Kzei~o($_;Wt#0EuW^B_wO_(6|KF;UxO@)QNdSe zrDjvV20X*xpF%RIaqHQ!fn#454-XH^=p6K-U_e??g8J(a{~Eo!1~9-l-B{cPw7=|g zDv&9EP(jVWcY&7Y(IQ#L)79~M6)->F?jI_@-9Zmrfm2GVtS1DR3Yzk=cXFpEial$v@nx_BssXBOwvLNZmjYVPTq zWrVC>i!O{Oyj(tEMuA7U=GCbI2IFcB!OLT!aBW^2eIj*uSsn0*_?-rsQBu4;-te$v5JTd05^<2F&q7gEeG`xR zp~%MF4{)zGL({GYqxeF7GEc*XE3$FPdi+=M!`P=nvAp_I2*4TC|#3 zi9)SSEm8M1^9DprWYm*m;?4Ekw=!w0JM!pLL-L~(J<=GViex)PS+(GwyOu9w9c2Hk z*&+(y(JTsvQz=H=;?W@!tHEt5IKj~_%M2m*GYbR^POKUyg)4Fo{*Kj$1O6y3?Q<6^4HJ(mQz$}&;W0dw0XAEhaxgd@R>@A)x2o^S{k!5Y zATg&z3^8V!R&}E3J!d_K-~-NLG@gG>N8f^=opTE_*tk1OzDh#^gm&bi39CY$$Gj5? zxnAga;QV5y4e?4gjI_n8oK7QFy)V7~=cXcXQ;mNp?(2?AjL-(P;vj|`BL9MW63R=g zGo+~S5lx@?@OrcG@b~hppYuKU7XMuu?k?uZFGS>!`{a_??2*-@-Y8PGlR$L(SYi6& zzFmT-ttUH9UNieyE{akSm67Lwf>>t^0?BHw*5JH)?qZ2%l+qxbet;f21wt^Iwm#hX z`yy7zRc&5u?xZ2df2T#}-}gq*AqKYMBB4qq#5u_{SI{q^;YM>Z9N%60GCm|MPkQG| zHifFu@U6%O^Ud=y9@lpaL-tG~ZZXi<1jmSD_H{RmF*AWAJF3J7CtX)RWHe^3`d*lV z;mof-Ex6>}Fc|)~$ROl}0&fRKe-bk4?Du_`di_-S?J~IT+Yp#lu}(7(hd?7vE-d{< zviI{IZeE=<%nMpVVb|nJLJ%^M`3JR?SBQ{?D%#fWUo06X2XNnC$|E8;LZYHtuC9go z&F-cQD>&YvxM}PR_kX=b=Hu;;0`GgF@hCtbl@9}L5NKD~#^xpt0LdPx2LEDIDFKt9AnpjYjd;`XoWs57(f za77_Q-sfw|WF;>g;1XUU?=Oxc>4AP2ZTzLL?^Q!@7yPu$f?Yn;nM*QJ7jlTU8zHLT z6axEyINE+?WEmob_E|yntCUbIzr9T8>kF9P61it>&Bp{ds zR}O+flVu3d;0Zx6;5dXD4f@^R4}VXH*t3z9+Jq_NH&*`a0Z6(we93%&3T=RcZoDMO zvq5B;0Iwn?q6$u0sQH4~?U7)!muX+Gs<+5xq-d2XBF~WhqWwF1ty7ZoOdLy(WnuX9 zUn@Ql6hos57;V`M3=Gyl6`lt37cR0iQb{XXvxXZU@yuAVEd>yO>g^M2=Uqpn;UV6> zNX$L>c43B&DS~4Kk{F}3bK3IU#)c=Tx#WYqPNQ+OG+@AkaUI(Uxg_UAMp+q z`&=HhVMb4c?j82Ld+Vy643xMf&wEsGkhQrlG^d>w%$ZN2Z?d9qu5`oMId|<|1>d>T zd>=6!JEi~P({=w|Nik@kg|e_ZXs*x*^y5{CfPlawm_7(aAZ|g2z^8){+4PYIC^2I^_p1LJ$K&!XY4{&H7UHE4ZXgWdoxX=!Zh zA&@n0At5p_`XLBNN6>%x>eH{|JjrVrSCB<=zH;M-E(iOKoQnwzkciBo#pvjHPqR!! z1Y~e1EudTy?IDJ`2$^}cZ)=s$BI`jFbTX!2!9P^ytw< z@!oSD`q=EpriA&9+nT8@?(dlTz44LYYJLX$kBg?fC~DmSPzZxYQ^l=r^$O5fp-c-u zJ^=L-Fc93cs13CK`9_Z-Xc?M0IM6C?v28mAcnzq)m;{B}cHmUjVvNlHFP$@!*^asF zS{ZUZ-zyQ{z3FBN@m1}l>lU$W4jtxf6zDl<&Lu&$6maM1W9H%qv4}N=;48m(pmx9E&XIio{OZ2vnYio8TeAWjyVz|G>cuKm~e; z7@_Qp1M=?@w#dAi9wq7k7Q@SB(9v1~y8(>OcnCDJx;Rmp-ZI_d<{1q;lYiOz4$cRk zWC$*4N@q!sHDCiEhEz#|Sjv6(u?}qm6d|m^Wrb>NU}tAGgZ4Jm8V*wJeg3Rq8nBnR zu9tvZ`vA;Iuin2W&CSgnsPYkn=3?8R6ENriR4}4O4W#1l$zci5Y}Ea|2jbrPphMOq z+MNODWh(W2;HI-MCUDmgID$yeJ;o6^R(l+%`9eHE&kSEQzD$~y9n^avGopnO=c7lD zFsSuf3ED+_#le$7lG9TsB?RteXlz3ew8tLpw?wP_+29~XFlbtW+Vkw~`9NzEhvFRi zO9P&jH5ZHC{a=NAd240D>!#0u_xsbp7+vT8D3dI5v_D)cFWTYK;BD^YFN`#pJrT^O z0;=lqdwbZt;v(++JF{E&ew7h^kTW2Drg7i2$oISBOk3JFT+JFR^?r(vzt7z(Bfg>D zD8H?Sh#j7y3cm0DsXyjm3Bf?4B9Ye2vm2y39nTxjsJclG8xna)#TefojG$TH7)ss? zHkE#3{sKvgUc>_`UrC$CkGZ09ED6s1p;x7-uM9;V^b&S7;xh)~Tbk$ZZfkzd6<);p zJ{T@si7xNEJz;lE3awA5=DGg|6$~DMxF|$SwK7;bUmJuL0wS&H!K2?a5NaK_`R>0p zWBX13lDO-_Pn-_lxv8+>kIzu+TqP6S-AE#A3?@DXtl3i4m!m|1KwZ1#q*{=!OBhlj z7m-}->?d*}YY_*$$eVcG%Lf&6yfcGFWPO>y_o}6)b`K)F!pYn!-Up9nd@u;FB;sYy7cI;&XC3Ru1q_?J{aID?0P+lb=*B8#O(B4AMlNNhx#ejg#g`?&O< z^-kE*@~Nfn@b&uleZNNh?uc7?;U4|;swKwvgQ83gy>3-WoeInuY=xsoO${nl4#8Ow zZ=Ce+D{@)8Gs+&(6b9MGIn7-}0UZ3WLJF`#ji(L7&&-8InA8UO;t9T{83{SDXh@WC zsB(D)R=XJ;MQ$9}UW$R+3%?h{MiR8ocpl#4MBEQC3B)(FlWx6lJ)mtZqySF*c`l&E zz`cAdFh3$^!|a3<93Anl3P2rkQzX8zrbQ=8J4{X2ciNZ*#^tNS<8RzCBkgRyWrFy) zqO8u3@=}w7>BnNxJer_VL3cfawEsSJ>))q{!YjNWz-SPh6ECwQ5*mIHMJaQ8>bcy% znuJ*^&(rXnH}7Uvl<2;-o=Df6uNzK|Qn~VU0%}#DGgZi=qFvSub%g5MJ15{l81;~= z(BakEo1Rr(M=&xcvj2QY(ZQ)5v<+~?ZGODI1pb_h;MW+PhSm4*@poOla4P$dAA_u| zmA!lP@Z!MZYY&tLg;~hZV6@x;AO|L#wgk3V@aJd4;X_*s*BkICRbO7s2CEmi>@kAP zo5X#;Cv@$z$0klj8`zp}R%|ceBMpo7S>c@S7N{=d;B^NV1q)j!D50f0w1f>YYYL8a z3(zoV++F?Jt@r>e99bh9PTsrsI1MQi^zWH*w?Toyxl72;M;3;o+z!J5OEwfW$q^72 zz)XsR!0RZL8K?~^Kb5n|dcrQowADIw;RszwL}(ZHKN3+1(E41LPdoSN3@_B}!NiJ# zq=A_n?5p5NOqmQIyTVLtLK>Hj>dJ1i5JaIBYy%AhjP z0?Gku)I+=NP^JluPlD>3XWwt;V0<*slks)QyM=m^u(M?mB70WTOi%jf&430{C+)&& zpH0$6yD`yHzB(9E(9Tq##v}gI@o2<_9-u#8*AyR^2|T!PcmF0>q(l$ zC&y*HiICur3JeaH-<$s%i=S64{u=x^h(8!ydx|VoyjREAs(qK6plc)nPQYOA%K$Pb zT;s`mBw*56U`-GoePu0i%D86d^}mR%%%1v;ajts$k4ff&Y|?4QMNJYW3@7}!V(a18LKPDA6`XOKI=#$q$9>^vX_ z{v{hQ28zUWZr(S|3xBu0get0&`f8P6PZaq{# zM!lf;hc?g%Hf!dAqS@sESlL`17NY1`-z0Lgf|I_8MLD0B#D6f0H1xYC&_J zFwSXw98@R;u7m#W0wi*Yr)POb8<6Zrm)914Sy075V)EN|`cfx3gv4L$fGf~|!492g z>d=9Qdb4Y``%J+O(E=>xA=+39JspK)T|Bpl!k_f}hr$dR@kdUsaD72A?MAHjMnEX8 zDY*Yp1Ox^HCfZ+S{L!8q?TvY6b!_HkJeVIyzpA>yDb+Gw({|m4G@>7e7?P<0>4@+g&d8UXS z^Bi+FdN^L~p8JZJUv;>r4L;X8az^*|Ub0RLsuSFcq0<_m0}I?QLS3Zxk&G`MUV}<| z-plLvRUS&oE7oqj{cmd5a) zUY(~C;4JfU|CGS+W}_1TY<^h$^r=EU0z`3pQ+(!$(|_lE&Lb8i-QUm$2D>o_TnQDR zKbw5Ia_t&5nudYLll6-7{%e;Yd;2#zP2SIokyRFA&yqu}Tq_+-CSM1miAgugsi;Fj zHNN$S^|w*3(j-{;9DXhcZOqBv*@sqk_`!e@$Zo2aQP$kmZPsK;6mEQ}g}X+oRt7i4 zfc0B`Q#Ff(85^o=YpsK5dl~(4Dv)nH7|og3jm9$hbtm4m(_4yH3n#65V_P}|rVQj1t*tF5#*l<>Hl4LH|!&SI{pvFh7mVf@` zYm=yX&fI);&*(Y6+-b*q%>1$M*ULqBCl7;&8n1|W_!vWQcKz;Piv0ZflbVLcCUt&$ zu^)a9Cncr#|4WJdppw`vu1kP?7T6q1Dg~T7Q8>JN|Gk~8L3Z1#-%+*L)?`TWYWH+; z)1Z?H47Z$6uSU=x?NTc+v;}a%N~!SAfd4%BvO#!mGAue9=FmL10C)mL4wmBp@WTSsAHB zgf>q?z~hMC6l~JYxAij>V?ZNn7$4_Adj&!3iM70*)59q%%XqhJ$fY`ha?WVd@gAWE z*JY?Xc>2gv?842~wmZ%CI(YXc>l)|T=Q8UVKucu1QW2`~y#EZQaDs$R`?_1hf_fx? zS^A+j#$a|7?+h8=`IRGQ8LY!o9%VSA0}&a3}Z+k1y&+5Yk4H`$|*os2u1%v4C$ZIh85 z8Ih8*M@#k=vMaJOBNQ18n^MXXr9`Asp&gaf_jUC=_4ysY|QA@azVAv2L%xI_dapMQdOUJ37s_9hrKg3Sl3< zcckJ!!@PC2+!oAY&?Xhy`_q_Q*$ZeiHoFS+1omLS*ov}rd^gRQCi*r}Q|E}jZ#8<| zf>e^!!CaFzGr`I?6@`@fxd|SR7vze!4G(O!Qq6ay%W@>X|N6!G;40lor%?WjKaQL- z0`%Y;i+IHLN+Pr^nJ^hz?RJt|FPQd6aBbf z^T?9EF>ZNj549mF;ysDBucF^IESCWhNgE;e_CWw6LUKRt5_3%%& z&#!r@gA1raWhY{VmR__Iuec-x@|66bQoQ1=>-wVMp3YeuB}|I{I5Kvujq?lL)0Qm? zN*0lxODsL9*WBF{7{AW0iLVb^U_7@=BNiFW+O=BVr%cIxXz9N?CqS3TyVcGn?P(g< zx7&tp=S`!Q%9lE$OLvm&UoxHjd2`3vn(9D*56szv_)s|+F=^7w{XN9os!?_u&sRE^ zzs(UX_vw~R$2Qf-V{@Aytm<<^umG=;6sgL%87*%rmY9f^aj+X#2XA#M z3JEsc5$v31f8nqn?~xD&eVbaU&=ub`yLTV$nk^qhCFmVGq>i#<=5diXr%4ervzrDk zUI_N542>!}#R~n}&Jai^K}FjBZBF)3_z88Tq#EaSyvUPe@L9I6dFyl>Q#_x&TA%-- zvTh5K*I6q=v9J&;we8nSh0rDukRNgFI=bt_je`!`m5ooMSt5iXY1+(2UZy>Y9;mX#`^QdsO~=tGsT@BlTW7kWz|2!lq*>hMtO)2m zYBE4|5e)$~fr^@YZyM0Ze4&z~63Atx*apDMMm09~hJoH;RlB|WS-Drjc$WjDsYuak zu`{Dp*X}7gy?8c&A-1|6zCxErYG#y4*W0bParFy3?VVmUuKAS1_TDsEN{PuE0_8Ft zI(Wt}eK_~jykycf&Pu!w?-+|wiPP$-8V{k1w}rRo$|@&rVo(8)Ol}b zs35Dtfm4)S{L3KK7mnMm|6m9?>!Q0=wM*!(nQ{MnxeU3JDG3Rr-8ZOy|NcmN+Yr$% zDruW!NaW>B7dTnQaKbMBMG$DSv(dC+`|lW`O+_L-TuSK7T-#Qvn{Y0<6tL`rSHqR5 zeAe0)^}SBrixYi!YxNP5mUqx^xQzIRYl;S^kF1*9of|11;?eEFU6hsR|1@E!#h2oB zZjLMga>}3IFK3L3(Ea8upjkmh!lL5M6Nsx6K93nbSWBghpxigNWo^f1H`=zVTeI zMb*eaZ=iI9^cSC+lE!1WDm_u3Tm9UYe94-b#Fi*Kcq?{! zkX^7y3n_nJeb(}J*gnXa9|6!=U_C<1%$GBau4n40 zn|2hptv-55#?smM^0*v_gZ`~@%nA#sK+B*ZyRy3e@j3IEhBB6`gs+;NmE@6n_V{uGg!hl2A%3kU*@e5iDrMSojT5rxmAr%&qbSsTzBol>B znhlV)`g2PXI=IdHuMabuM=3hiO1Fjbqa0SPotBtQa6=y1_21BXySz|Q6VRy8cC%S8 z=WM(MaY}BO%8~*g?)Xomm7`})%SvP?CvBu-CW;%?~l=#!_vI~{H2=dnhUd6(=H-W)gSV=3A6w8S)Vrf;oid06S zEi&(CLm3-|gCPC2J!Ezrn~!u8{e@jV+(eV$So6kp?RVwn>Fp=y)~{NN8ss+WWBURQ zl1m;;eHl;Y=m16W8BVfYAp0v4A|QaeZk*gABg2dnZ?wH^A9^~DSA}HKmoY+k@qNS8 zW|^KnWZ@B)L;2p~7Lg#k2|hX!(r6fcG4$nB#R~1cLSJ@P3qyaC8)3+95_V)z$(>9G zhUh)0!i-Z8ge0IXzncrF1EgnV#h_%;rnk(Ws?V}sjT00L0`OvHW|r)d%QdjVRW;Wz zjzj$m(fy82x+Htni zh&P}6MZ?BIlsFT?`86V$iLVvq7`)8;&^dHHc5lbCr0t7idj4H`#e(95TuRHq+k2_Tl?LDUF8cUv zb)cS;*d}s&O(=_L&*hr6Y$d1KHye`4JkP$Ydw;w9q=|jzjR)VMRq!6=QKEYC6$i}6 zs?q^GK&Be|cdKz!W+5Qx2E=Io<4)bMVyY^A`G}9k&ba5&CxX(LKMrsi68qG7eRq3r z(SX4NAHLKHfY!L?FjsKc;RU0sKl}HazT!(Brw^js=&?wfUlb>oH2*%6a`NyH91TKW z28@^|B?Pz%5{C<3spYSi2L~?ENO|%;)Zj!4c@yQPPHiJ`QY$LPr$A}5C_>l2&l|QB@ zgZx5Ec<2$$-5e!k*gjLnJV>?!lo3?XV;6q1(cb+xe*P zjI+V3NSo;;O3wW!3VQLc9#D!;{*f2VtV^1abO$tiR)x_OSa67Lp#mnHVMqv$qTUUd z1%1T%<>sQIjymSt%4=26t%6RnXw81gls|Ka+WWKq$jCa&vMPhUMyzPgjGb{#1hJ1T)xi2C7DO7GaA zVFug7lZN{NXq{DY?x3ZmZTVaaGC6qqBog!q(70p+Z=&_8+gxw&5~-Q()%dc0>jB8eBVU88S9 z#T^lxsOGNlh(*+OFj4G`*=UeK71!O@b@qxtvh|T)gGGF>UvZn($?fZPWt|5Py$teK z3CB6fC5`);hh@{jc2ja!f?P&kn2kVRP;#8+L$Ww|OYeVMa>=5HLcV`vJ--B#io$7m z(Wne;>tAJXkPmGyd*id8^?{&N&ztVWn?*X(eU+CbpNz?Hm$vJ-IzIvB;7@MwoB>o5 z5brY_9HUJ%3wX>;*Yj0kN;4GAMf)cs6|0CQ*hJ7)%JX%9Z`D*k zlkbJ2hVdq~EbE8JY3T2NQ?1NaQpC{~o8V=&%!9j@T7=6eZQ|@SvNxxOsaeCf+ec*6 z-z~oIvP<&VoIo^iSj7hM!s5}C=bQfQv;kt74uCw+Xy*594o-O_D_rB0TuSS`i!7O$ zkS;KtmQ_j09eV1uDZk;1DPOpnMMQ9aGJtiD3g>vsyO93zj%)C*aOk#ju4h;MlLMe? z{% zc5ZMiU?Hz-+5kBlV@sQ-OMnIo^w2Bhzol(h+- zA5|Ma8_koMfDD5Lusx+JUZnljno6xJgb2~UiE|v@}o;9R# z;u4B#m?-r>ctx^IkN)FzZE_p=&!~R1juO^#_R`wlv>K3n-H#iOf8CDdFk;;Jk^0~kMxD}UBE$+yV9JM0o6XJzG6nb;d`;+OXF!nURwF zN^9W!N#Dep(RB@CrMCl~P9?fqKiJ*kYavUG`RQuy$f+Nitg~dlLs6Hq_;;R~cn9gv z->lPev(X@#l!)afi7?bR^%;qggA!L$z)+$-d-=ziVUND+XN`Tf*(8}&7|-XFt%_)I z6~ooKu&5O4ced<&>nTcJ`g5`&yIA`wXv>1A^c^WkD_0Y-bC2q6^p7-_La#4@^@DCu zQP=*ZS6tO|=ub&)J}cSzK>2*g&f5cG?oOEkNp4jcyr(%L-!^E=oneXx81&Dm21P<+ z0CBsj#5>D+Zod4irD1Q2vku%-ElS zJS)*nDucoH=2`A_?5(T+%YG9o{g0IL^`1V$x&MuSUG-42v{Bv-mKBa}C#7@egzd%q zCj;6zcPZVfpYds(@D>FW5o<@Wd>1F;n|{RrsiNPIGSvBBgd?|YyX2syZ^XKK*6z_u z!Fjef@obuWKe#x)%9MdqiiR$}b{r5H;WHz4hE$?A7bG>5H8Q60RH{p8Pn*FWr=@); z#4s_oCna8``+JWsklcV{ud}zR`B4D$8$cWx);1>UdO_*`NZquuYa5{S3lK^Y}DA?2yn;;PVXZS{rAT3}T{Zcu)&azY)Wzr~HFP|V!4iY-<35>R6 zR9sP1h*}Xiac(z=ooCE*-B-T7cwH^;Xzn1+K7G1OA>|cEqNQ_bI{%9&%lwIz3xB2& zWuf1PyJ$;(v2vbet#Xltb6V6bwf5!Dqz-dA{h%U_^*Sr}M0)N{mnO4{D+4|Cf_OYkM(LI!3C2s09p|cL7#07Nr0fV&#K3K zbwjlS1sPYrS-%+ittUP!gI)D!3IweGHy^UC;>MGiP3rRUln@IO$=CYF{1OZ~!$U;a zdwKu-Sa37-_vAA&R6%L1UG*ajI1u!J?`dK9$l3Hc)1SaZUUF4pzV{}z^5Geank~gc zt~xlV>vy`Inc(atQ@>!!*IX!dpq^4- zwfD80AnJ<`0Ka#6eSCmYeL-|L4g+vS38HnvU?AMZIv_T&nLyvCUsO z9)nYHSuYKF`^eqtiQbB&nhy`Z*%;ydkFkogCq_&%V(D_n5)(?0r{{roNyy)k{OA)U zOiUwXF%>NMOSLZZ7t=wy6jn2!(~=9G-=rMjd1`(|HG3?eoaM}p>}QnXzPI1jj_iw|)8haOMKINBSUj?SQsNUBNn++KoGIOG3tRK3~ z+_d^}xRGXjD64qLuQ=OEj%$y-F-_ljgPF&Nh7+a++)Hf#R7}_p3t^X;|LAyDt`23z z&Sqz+wHw6kfexo`?P)yJU@W4S5}jLJvn8+olaK0VC>_ORiVa-6sxX>jNb`A)Hwn+I zuuX;IwZHZb6RX5+e1BH^K)cPpF9Xsw+93|L6B zuINcMAKm`M!vL~)UuQGP{U(K-mfL19hA=D-x&j!zd^Ht z66!N{KbQiB}9prhdj9%~5kRf0G+JuAdcECJEgxzB9Sd=Z&rgqf7He3?GB%KZ`?yH+~l;%c` z!ZzO4d+nL7fZ7+%55-ThEpcz%Pi6C%8h^biRb%lnvNhbgx1559N%X< z!*uRF4j1iT_Iz70-Afh!vF825*H%A*+c+^`^KbWj&NNb}3~bpbV#=}<>gYq+_~$$U zFjqBJbhCBx{oc{$j2_B_F(+ zFEnSed(@UhAN0(XBVf(nU>HD)#M;(@bqA;II9=){eU#e+tw}h)rW^O9C+J~=#O}S? zfeUHfS=s77P+8Chp}{m+(C0DfA%9L<(|ku`OkOc-qW3w+|E?;A{h)YMH^Q^5?n`0E z6T}OtD(wY5lWu2IQR=7_4~QRa*rpXZK%$MZX$(obsPEOYsO(8}Sdez4?{d@3qkzQe z(r>|H#Moan#NHxZI5?US`2N463$(rR?@$vjru;IU$`^f9*noi+zr-gV*!?j)K+!$w z2iw{U6rcKkL^cL~%WlH@K8b$CSfRs7ve~tXPmxSgbRnzz(HLSeO4OP)JRxUiu)1^Z zO0HN0(CGAQ1StSw!~6$j8Hnc)%6(;Jq7Zeu{N{^&WWvDO(br~@HN(YDH3o-MSHDpC zPPbM&o_>H`=uLc#0#{Vl!c#R#2I`@l{RF~h{BD+8|Dymb>-k%t`zUUGp`*jQzGu{^ z^7!wfn=3>zCc|b}k1am(l9ZS4H`_AtyXf}lIDrQy5EIaF)-p9^fqY>y3=-%Gd`!=? zoM7#GYW9pR3i-4+uUb^r8Dz8}i(jS?ZJihe&7SybwwFas$!2@0`tR-sZeFT+uYFF< zeV2c%^EtR8L<+jGT~M!5k5?B}6nok+w`Gc}yCEq#*$wClX#E1{by07bz*$|q`jH7r zf7B#Gua&UCdj8x4;u{^2!yN3bJOJ4Ogm6A~qX-Sos0audE)MbTW*P`&mCP(#9g8AS zti_-ZEmQWQeGhq$^wF1k9aDIV3igp`Moc8)#D=!_v|%JaHYuWJ>@q(9HaVtN!#ay+ z60;E;B{?6HxI2;|A{UH?ZHo`H>mG5_JW{MqBv2+-D!2WL5+?grY&mUdVH1x<$yqhG z(3Vtt)73qXX?h`#WpjD8ZCb&soy4Q?asjWw%OQ6M#oRx?<&BEG=|&t%j;qIYo!dEN zu{LaF@f3H*B18uLKEJYj5-_;2Zgp(Jz`=p@?vS0>fTj#KdsJVZ+l-KJRIhd{V3c64 z!QuXK4vTn9H5jF{aBKm_#6dW&`A)ARDi_VGa#v*2x)Xa$rofq&b+ z!b8LUP5gjT-Y&~zBGb&#&UT7gT^ydE`m|OyS1rVwN1HAEU!Z)0L=AO&qqK#E&BG!m zZ)-xfp<87(v+Jvc=1HYROCKr^0AC3qfDU@0V-Yqrwkr8Qs5+kYCVzl{vZ^8^$(BU3 z6Er6J9ng&|siZKeH!tt7vT#*`p{e}2VwJF)7f*Vkjr zQv}_Cuor_q`C7t+XYXJmZF)lGVbD>?W*VrpVVe7foag9JnlQy70|8xx@}foVF*Xih zd+X)>>9G~F*nfOi|6P#eVU(ewgK$D z{?5`8D&5ZrVMIba73kP2-$I(*B0)huMyFWo-1g%n5=1O}pce9Da#|+~$cCTdyvh63@-7rs3kQ)R98W+e_M>&&yMNoXQ$6lcKF zC~|R^{4N$(vWIsU7T6%|PUYMsz=>V+{9Wb=PJeGr<= zerE+Wj!Z$t>EQ5zkMaztcDl8v)&HYZS=R9O(STO0XH}c9X8Z-`V^Y_KQ_wz~zgye$ z^9eDMlyJ}PPO)NXK9rw3Z9a%V3L~oUL+65;0l7)@w_<7#Ep$*z7QHqzOV$n-Q%ZIB zn`zvtUMFWWE5Wlt%`nmL1g*}BC&SL0BH3F*E-R9<_PuG)yuz^@Ae&Z7dzwxrhot3H z&jGk=DOi}F6R!L?XH>jA^p!oWYe+*QeIEH<@hKzO zU%4i1)u%R$nbS3lDLSK&#wfv5<1cxUiNG-^T&(UC9kp%1Y7_MDe8s;m;gF&>bYKGQ zeMUC@&7v|s<=>_zdWI;hqn7JgD;WGeuGY_%Ly@@hHmyV7Zrq^O4!4S*xMJt*+tCon zFR;;K2D&o_cdIgN%Jufx?ZV=}oTn9(%Bb@&uj5mia7B>+veN%P;3%4K7B&^}07Vr{ z($r2e)D9&ieJV|p7eleSe_=aa#4BN1ls(ro(`sSFdDw2VE3*LH{eX+8qvd=g7x9rdDF7gBJ}rR$B+(WTA8Wi? zaHGrrQsw?Pth07jRei6Y@q1)h&mi{T@Nf3E^ppuvD@4yL=NG~`unx}|PQ>r1U#R+< zrcoyK*l0#xrBqg{6d@yRnvj*Hc;O1Sfr;P2l%n$qBa_OEtA+KUik+shm2Co6%u;fMNWd&cZ12Ogx)aMKA!M5A0$)D268>%GtG&g zjQO7Yv$jqdwAXv{9hu4BWMntSCw-c@Wt{)Q3nzQyR$l2=&${mBlh(~US_!F_{5uePC^j&e*UAe9jbHC@+r>EI{qR7^i*4TP zu>C2wS&7vD-G?_+k}Ya4v23eS&J$*c5GL)_j+pd_YkLq;-S%=;`D&h_2j5rrqIy&3 znTzR>ZDZ6C!;EkDG z(6I&v2J2gF&n5JVbmBmHk2D_qwTjaUxx=299Vfv%w)-K--;Gr3`1)mFeh%r2a)7L@ zo8I{CwFin_``)nLDuS*j>B_yh1oJJ^Y(>9eQ;0#laFoMqi;u}+Nv=-@aC0lHp!VH9v0 zbc_UTkf7bw1d`%5wU6eiRPz&;5FSkqy&IpO+d21Jr6yqQ18vb<0^Z>06@d_9cFj}z z%#pcn9IIU$%_0seFe>E9-h7tLkw{83^!Rqep60{16CkZ83~BmatFXkg_1taTe#wv~ zVWWAWyx0Cz&dvaN&w|bb$7_7SM>bqOvi8jvp0=TyH!Nd04slY))tvmF5c+r+8NojD zLwx&F_VT~Hz1L)^ap?D9zSH9{s8!=32TV%wKZZp+_FaGfE4e?Of!dW*PBd%5OfFkX zE}JC1ys+w2HyeB9zNLp7{^j@^$#QE;9dbx^-$SDvaZ|QKltJN)+qWCS9X{Ny+fV!| z5hhX7oc(q8<5gFq?X`FFX$KqRQX<)hgno6I^ZYz`&M8Tc9J3rqK49SbTZ@Pl7D$F- zWO1h5x0cF(p@m8Q=>MBA^#7N`v6UVD)J^zSkpNcaG0?#PZS=f$XxGJ=>j_Y8`^`v% zV7HS1L9}hM1>+OkJm>XsCJ8hyJ5G!b_-vj51%rWwCB{&+?b!@20$plh_#J=~xxicO z`|O*K=C<60j(-XheBeA0R`6(P~eUYO->l(C|FZ9a( z`{D$+{kJ_ExDoBMOiWCKPXR%=Tt&yy5&Y2kq3a2+9Dgt9dlLqImX?;0P-x+X(d^}; zyV+@k=s+i}MbImnNhe z3AvcKHA0h9TZd$4=h~ z;LaUAuyX}Z{P6$$?y1mkkx_ue4WKIrN-JS3M98QSOn?dhvkraEgosYd%hLzyVR=Ys zy`<)RgE+0`XppH6{ior=t7+UKB&+*@4~Ba=FC3Yvd}yp1K+Fh@ZBwb#SlO8 zw%ZF%2Q`V9BCkE4Ob~gw2@}Xs%*@QRq^?`pG7F1FFkS`zYl z+}f1keVmYoiM88(xEG&mx$`$a9d?sY0Pqhggq_-R?yl_{w9g7b(c?7!+bl%Iuv$gMvTaV1% zR(k7$IS=7aVfYKwhkbCriU|Xk!;8-Ow*kZ5|lxy_g%eiD9Qm*Je6G)0yce_n-;?%69k=tGso}N$?Dei)7;a8-VbS$eULig zMsfDy@Z1+-+#$o83^|LQW>^?+{_jrPrBT~~p#OXL^=?*GC=&ZJl<)Z@-?6q2ZpZ|- z3t^eyLjr}E$!f2(B@@f1*`pCCn#kG3ZpY<6*ahb3A-OGD!d{~UmYWU<$fNJPLS ze9WI7UlNg)P71yhOOPc{N??PrV;^k9Nu;mPA4tThmGFZKgB$St8xt5b9DE8UB4i_0 zMeH05I~5T@0tp9S?dPA!t!!)C$rqr}OlmU(3I#3#h>V_BRsg*L`{zt@NdRkfXT9+*?}q zPo9B!s2hSN9DxEdOS=&QH;~vM%P-FmvU`(Rqr;7u>mtg^#OIjpjMjpc?!AUR z2Gf!Bg7pWNVYNmRt0MwEsuoYYA@K;gtTSh1pghZjFx8|0_By>4wcW^!ucNyggX=1UN=%XiPDqikGW!?5m&1wrx*BM1u7A%!lo%VJT)QqNHR+ zXrd5)^vtki>jJ&=StCLbsJA?jT!;nw(Hz{IHo|T87+mc`Jh5SI#5T+Xa`0M0SOKqH z8wAJ@2folIL`>}}YbJOAaT2=P%rG+}EW+l?BOkW~*MQX~3CZ!@*bbXvb;6%=D;PNf z;fl_jrm>i(0NX^Wh=rpcpLqmNv;{{&f*2)7;oE0bcfwhcFx2)66$q8lt%IGG(COeY zR=A1JpctZUWW)$ZD`^N>KOPlP9R+anptBZFGU&X>!4{f9?PXREnb$&U@>{3(-AS2S zVNt|7M3O-Pn`)ui&f0nkS8Avo8XjX=!3!K(d^?ehjc5Zb00@xY<@R8}KevXx7sj9{ z!YiDHj;`%;O1m2DxCx7GA`yY~JFuhh{%(|kj}b9JN?}NV-!_2z*gAoJ#GcO6DhK?) zS)@bRZ6zdVer$L#Fg*PFLg7=uKq0$)t#doE-^Mc%;+Pp}(}a-?GCL3z{;yfuZ-bWq z>t7F4?BB$X?8Po#oATTmnesrz1kCHdF5URG@J0a?za)Ibw-ku0wx2uT?M-;RKsgyn zLRNNmr-#y`LLM3_q!6isCZ)rKtpW*N-Ng^**7IxycM~)b%IOoXWJ8zQvc>U2wjh31e1B-t^VYqcZ1@ z#REkiVaR3^H$mK5{0#}9PEWO|k;r5NnB?|h8zO9g2{T4QLc7c;lW>gP_YJ;U#b{76vrI#;r;uBGdJos=)n=(>3_Iv2DTXrG2B5;d@Z#BVT z#C#p!=WyVkE3j?fb*27`|7(c+Gz_2fRLWm9Ny~1z>_+_N9W_5f+?x>bhf{ibVd0^< zq3ppIAsbG#2hYWU6-n5*nh6dPUN0e~deYe6urlho#aCzPa z7fVtE%AbVEc|T+o;LlCye8Z`53e&QisIDUh3Cqc?UMp@&m~kGgJqm?IZKxB$aRB=+ zrn4dp)^JvR&Q7>%IY&ageu^+{CfMh1ED{nNQ=L0->Z5Fr!}J*DBAlge+_+KGEetT% qJoWDrsG<*{EKB~V`2X!B`fJT1lFg_S-|v&~kCEOEN}Y~t)c*mJ6?wG) literal 0 HcmV?d00001 diff --git a/perun/ministral_3b_aya/training/training_loss.png b/perun/ministral_3b_aya/training/training_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..efbca3173520d8e5086e94bb65c0a2b1f2152bee GIT binary patch literal 29717 zcmdq}g;y5q7dDO$Dbgq<9R?vK2uO?Av`c!q`O0q?ylb+ z@cq2+`u+*Ov({OM=b2|_?%DT_>$>)y6R0SEmk6H*A3+eJ`}bs&5d_l{K`_SfaN(Ql z?L$-W9}(Nz>b8$8p4&R;TR%hO^ldH8ENsn;4VdhoS=$&}yyWNRVRz}kVUoVC37#0Eh~^wEDY(k0W35yZypzRXP($C$-o2S=61wYVz| z4=z9JaFLe2mw4G0TjmW*{i~#jB!SycY$?jhSjw_;KEKX+rhU6@*iO5-C_(+^Z7p?d znNRn4q}kZt=v?`5KQ(SSj5msq|Lpl|?@E4@SIvyKm)J~C|7wXXvG`pwL+48>?HK_7 z9QbwgQZTWwuq1`nBll!vWTswFBZ24vLM$dSGP1Yv`p88%yv9%H2}fnz{V?9Z(WX{j ztoYow%SOld=5rgr2cB2b&=^SVEfcv?I^i_Y zy#49r(O8tOd(HN!$8rJV)?8Mu!C;~J&PJz#=2gRdbzbuUfrGu?ss=fs;i-aBTLETl zL9;$jyZN5+sUPnGLPF}B-m>ypjEG@lV;8&1-M)=;wA<=ou)Q>_nAQEBS08h!d|f_m zO&afD^!Sjm@@UVrtGAc+%9S&4Kgm+N`RQ*86*^%Bro9vEt&(@2J-Za=zGobKfj?Dy zbGX=wHIiF+4ic4*YxB5!$O#OQnl^yqYoWC$wAH)<**Y166 zc@7K2g?l+JzmGC!2Vx<~uXRVn>T!*ZT^=_!% z`gBWnWPhn0XXWlR?Z#|pqv_^~Z+r4x!+~$ch-cYSDS`JHl7{~-J#4G;@F1t65{)<> z_h)R&QsY?|C?F^rvL<-m8W)0fhFI@+%njl{KBVvY3~?D;S8fI~h|^RZZRX}%j7ktv z^CEXc>8#kwo;6c`N=usz;;EFfwG{v(^iAkKGtaO9cYqH8x`%UWjuU}b=#ElSQ$@ln zCPPJIeu{1N8TJ*L({3#e`hWU#F=OHAaQ|&Ygl`0=n!(!CkK*~9GA5tu2sNF+(ZeMN z?)l-zt&C+kM@IucQM-F3Y2 z2cBzN1!vy0t}edrts$5%bJ>0wBj$82h>EA7E8`KL)r1T-4$f|I-49b_WS!$bsFlAUMce|UJ*Z!N|U^)q(4am%PVKj-sj+kxi-G`BT0J~sXO zW!op=vSrquttq6S(wR=E?6jQ_nZRd0Dype8_&#d}n7eO#8B~!<@-z z8Y$*61pG>t?jb$_6*o45I)&p(&6(J(eUIa9kN3h>EHt#Vv0GTka^*gEszQ9|1^(xH z-IjN@~@rBp6%=6W0wAbENIL%Gv>p2M;Z;@NFdKWI@EGTWJE zIQ{EG-#6oT&4-cU;bR|O#MOidjRY8Xrj9Fk?7eG_oPBpi0d<=1;DD=bw@8?IcyjAY zOZO|R65$Tzhg*a1MIE?P<)Z!WGP;qwY%R>@O^(mG*88=B6<91eX1 z(_EJO=!4#Kwj(7yJ-xW;?aL>&w%TOaB>}7`+!^S zLch?o9P{vyjvLZjtJr$*#_{E^*8bvmOpVXk^Lhmut9^!H%zS)groB1!ZiwAX}?!1B+ z^J0|~(+$fpL_S_((Q;yNnWyCVXrGLMA@qjL^jKQFd+o0e{P~8B#N}&0In8sj+~P^Z zRuE}vX!rY~;j)XGjjj(9IgB{X++N;i znkZ_(wk-m0VPo~a3NX1pGl>E8YOR&1jYt?@w> zRd*SjumN4X=YK&An{ur8>pfpzfhm=a(;te97cc5`{#_bUo9tbQxhb67q-RU9m z$Q;>N5sEa0_=d?a$L6jrWxUL{ zJ-uq){9<6RwH-c!R%gJLzI}atg1z1EhIWI4>291OpuAdL zwp{`!|>V%A5!UHdQsJ2YQEZxmuR5iAWCX#Gm1r?p2#W~ zS~$D{$^|4|jiBYumo=|&Fg5+B2~eHi&G?4-p9u$U?$$9vU}T(FHB-Dy?8D&OXm3K@V2F8 zp5yMHDEiQ%ikD074H_MBhD(c+gORP@gmS&vg_2^N=XfD(2q4wgh=ls*H@jT)$jV~_Mt6!x=u?X>CLnwXF!GS9P(4w7E z$OC7NcN!%vhwXaDw+1a79CH#AG5-Ad;|SZj0`PY-HTq@kmuJm+mgCapM|<-f`*~rd zDp}p~`$biZ?|#_cU(Vv%j1@Ge5wsWy-e0a_1b|bw>~ZV@;e{0u5y4eR*;lc@R2K01 zbuCxX(8Pw;%a_~?ZW~H6w{MR__G?6=+G&0IzFmo;nxuz@iw6M2cMMtGCWN$tg#ILq z_I*VUQfgPm>#WAAF^mU4@W04|;2nZIg-1jbQRwgEGX}ZJ=WstUFORg$VYy+kXf(9C zS}H>e!at{MNyLBe+orl;TDM7}a>feWQIt?onT;Hegk*bf)u<)Ky=QPOPNQ?+Mv?oW z^AM!o_ab&2P`)mP(20Ya1?>+OTP5ETnu2N*TblR0<)iQIAjili7FiQ zm5+vobzQXPmdm(Gq==B+-QA|9CQA<2wW<3^HJX(HE=%EIBpCbL*Vd*-#uoa&DrTr4 zmiM(o>|eCT)q``bOubVO5vQW3U$pJ0|Nfon>QzFxjCp33_S2^qA+d0mRN7lvSpfjG z%A{?xWj1qHD%gi}EL+95#~?*kL++aDlBI>JFgIs1TY$lwJ|#HK70twc$8+aIoF+@oo&~DvC?V%d5o|XCW*vOLNl2G zq(&(xryKCZDWl!MnS3C-pcs-`=p`L4F&$pQe7t0Kg{0bkKxpV46c{XD77?KVx9c$6 zuehHhas1)K2jpWV;{sJKz=bqxs+Xqo`QtM-SN-^9cOLp3xQDzRhZTYxjZBX2&dYt^n7U<`)ivPj?C(Hn6)bUy>j z;@`qotbivtRXx~UcGvS*2uJ(4QFv5QsSCAlY*qg#G9FAWp)LfCI#Hh z*k{fp7Z&yyyF7cA(!7?VTv8Dy7~L{k0#wleU@WzSD++ICzA(7$ZPX}89!_oe(!ir4 z9cNP&+1`Bk!0xsxFcZkb>M%d}Utgd4Y|=6ZGH}L2%P|++VZr@i^{(tkfgiKCEL*Pk zj-^ED)WX$LkiPEL_?{8nhxCR6PV>Hp1fNO>iV8N>b^Z!f?Gh~& zPZFCAmr_U`tYAs(FYu!j)p0(%=#x#oW@GYo^TW?Yo8sERSvEO^g_OCKdmJc#Npo2= zF)`5?9z_A%=7QL*8+F%LU_F-$u`UTVZR;(e(=@F4i&;5Ag z{r!y|yS!xJ`TI9EWVFQ-x#HhKg}bVrhqOu_i=hP=Op}z8^E>fUj(B`Yp;2=|VM*Z> zrk$RiUXX4>e}8{KaIg&cCY0X336kKzz?%>%M^Fk2geAcymf5OV%`=q;YY%1P#QL96 zU#AJ_D|6uSBfP+3J_rcsDkTAT*}@mW?o9(IJo%=*+Ys3`R8C;4afd`xh7V%=0)K{HR)EOV>c zrwTnqmqZF>@v9y*mI>*L2|ruf60bz8~(Z>w)H)qkeIjuGz&g( zJF&x!&cD66jDQ8{XKN?3C&%TeTokh1Bnv-fX5y=6bhXTt8~FZRCMv$A zhn#`}!;<*rl4}F^pP2$*p-`T&qZb42BQEnIeIt9?_zE7x6;FC2ziskcLypk{{J*n5 zdK}7Axr2$8HIJsk8U9|Sp%$E=N6J^3Wu@(S8@0m~Wl#843yt$LS9*^uG3! zl9GC+#W~}Ty6_pgUTXWBSEOHLQUwWdMr{D#-deXo$@XF+c$E6a#ul> zvjj;1^Y7kFYyF6>3o!n1uQLqQRJyKS;LKVBmCsvfKI>G_Y%TjIBrqLJ3su!*KF*?B z3fl#fE_F?`#~C^Ss4idsEede5qYzT~Ut1C5t|Fl@f1%>_IM2u`M$LDTk$yWnj&L1~ z(%iJ-VrrmPvSqbB4|yu41nwW8b*Ae0aQXeUXZT;fe4&f8fkMVLB6bpO7P@B!HLD6& zdS==*dwY9eQ#*jHqXZj;2bOT}U$LUS<41B9Zqs7G&J$cgbki7a$08B)EE;hpco=#0 zrnU98J0Uc_Iu*|MA4&E_hcY@%ksSb`tp}Ld$V3LLkCIHm{s8-ScBJ3MgRT3-S_+rNr{b1{vLjoH}QYt(ez{*cNJWt*C!)&5t! zB;)!_YvWjrcROfCARv{|eQKjD8n&mUp`ih%vRrQw5*@-5ciXe||I*&xzOz)etOyhV z_8A5BPo6wMZ{u;iuj^kO7Z(?bW?CRAP@1d%MCpcFO*ZHQdUo8KO{3o$bY04riE|-F zP!~4VTzA18P#(?8K5Yr3tv&;!D`UFC=o~+L~OjkvI;=nl!s9$G%p5V z-wtBg2GpKEa{|>VjLkki*eL$`@#6=|hEe-N86~6J#g_SmD|0j47H-59c5db^DXw?GvJ&j!g3Cu0U59o8K+mQ7 zj%9Clipev&?<6+6n}K$bd^ao!at$o>Z+o)8z)tdwul>PLmlN~z$=vq0B9=72epSCJ z?6`vJO8Q^>dF>a}e?<$C;NakJ_r8k(`dN8=O(D_Dk#s;bB^rk-THR;&f+0f2)2NHL*wak+npg+~Q4e8<^YL|Iw+O?-UV z(2$n@dgHL6(?TC#czC!TY&>zUDM%V1M(7F9pu1}Pn=4^+wCGBxf0jtWpRexh%Rhd1 zZyQQURo`wc0ddFpzeNlzTqY(aWU$D>Up1@V69Y>lTkW9_X@?UXC8eK)+g|a)oT-iw z>^GIkGG7rRmIb|Tbw%s9*tA^l%HrqR3-1fJZd~Q!ss}&fGx&}dRYY9!2>7RrX6yJg z@a4X4(7Uf)I|o~9p6^o1wQnk?`TMZ2)QHrGcy1p2a61 zppPWnxI*Z;s24w8o|o4)-%O8ok6u8+`n%Pc!J>n& zlhJ59DQwq`AVEH@4)U=ggaRZ5y|HSq${e)E^j2-xj$jNRu;=H`SFh?;(p^!A`(Cj< zf-?7(-fndDYaCVP7@r{)Q8G|sGc(bxri;@iaMg8rq(UO+0vVYdIWsn3tIw)gxmnT- zpOiBaRI?8E7^;F+2az36B@-Z6JoY-{KbhnEV&3v2T-8p2CDy%{y$mV7PLFq|lw%5q znVI?Cg9oqfB(K^%e)_`#YYY^8zNby3e>a`cV$BhOpL{1oy2|}f!!@Mc$fw;XoMGO! zG&G%qjLhkM%kw(G^*|DVTmZeU>q+94#>05`!)I;HI!3~cM%-m5>!Q9km=6dE?KU&J}3dYe9wk%@bL?E@5^Hy=N~1{$QR z<}Y?tSeP0E1EZ<6)&Iv+X+_01xmg!Q-Dwd}dO{h*Umxyo8&-Q_AfD`xQuRj4oj`ZZ zX{L~%!jARrPmwY7=YEk>EW`eY#jFL;& z`fzY@S$KK#ShMbf7fe~X(}qFr2>tkx2E-YyfkIF#*~sxX7WzB=uY;4VmDl-`pv30! zACBO@n@st&nET-W8GY@><~aD;$cMW)a?M749!{S510Q|tibKxKaw6_tBlYqdEV z`;}B&8a^Nf)1N(i7VUDNjR)AZTZxKHmoMW2*Le5y7`hn6083D<7aAnLz3+w&cw;E~ zRf;GEA1_?3e*ak=dcsM>2KfUkt4sX!M>a1AA6kiG=Kv#t*22z=l!e z$>ZJDrr*EsC@GOa;y`~0G$O+c&~ZL=Cr~#c^t?8)tzB*X&H4ofqBkFi&)5r%#Hp!j zFRbE&wdO?Cc#EnF;L^c&SfP8`hudT%91It+N=QICdO2Ha21z znJ9g80hrGRaxygj{?T+P9i2GM{nXGJN~sIZMg|Jyh0Ij<$zn=k0@9PyA3yj-9XatX zexCc?kEw6gfo|EF?|!USt_Eeh>tFjoN=LU~txJ`xoDoxamKi}9Jr4L79LF%wJQt;H zf1$RnZWXqYXuc=g#&oocj~aP)FUxpO?>6q4o#M+fGDj&~`oE~0x0gZyNs5>aLi6rY zi$45E!K^Q@Mt=}wi~;RNB3ePcxvq@&;x2{X7J;5Ej_^1|mbcnUQdlk#G?|^3$T>?! z_Tz{6_K-K!`*J{hAP=)~b2mU|!b{Vlf&$#Ze4zIg4ndyBvHQW{vWI`m^IK4s;#`(W z?0fmHU#AeYUyyq8Ep z5fI$c(4e_??OLAe?wa5-CnqNX+InyYB*Q8oKn{kl1wogv`Y=aNjlq3Ke`BtjfSsKk zWI1TrR0e=jr?Ym{BuZX8Vys}E+Br)~M&>mzpyA{1?_yS!ug(07WqRbT9(Ln-DleN9Y14Yt_QU(S|w|1~R$Hw%0{QNM{<`13fd8@}%bLw>1 zEVt14j4m=DS-j30M#lJ2C%tDNfUcVCbEFUlPnRQvG|<_OCKoiSp%^7sR_d5!vmRG- z>s|Pn)rEyzrNfbxk)ih-Z$0Kkekw|jPQ49C9yY-y>74x*l5 zw0Pz6moL{)#0~Y1VYEn64}SP~{gU&0SZDfqz`1qk4|%CSnWvI{sEMZ;-giugJ$pLH`-mFcZFfIfv4z*Fiy6AQ%}g4V4h@ z3($n1M1K|9ZLDHqbm+){9|_}<{Rgn6XlCRL4VEI9;@$xwM`h6RMg+({W^crnk2jP0c98hgI-C|3w|fsOeD)nMIki z^;kprf&dv=zZ-N>P)ZH`40Lqi^OLW*XcyVCvdr#aXEhxhTUC`Z&f~}lEsv-w3f|e# z*}1w>M>1JYr5nCw3``YN!*Re#w}Bq$3(%k~d$hm&GU_Zoek~+G3~QsT1@)I zkZTy{$l7mI9WGiVcAKoi1QRIQOV!ZI&a<6WM#To@45F zoEt4nd#~hW4?dc0$kU z5hPq?Zxf&hCXK=M3kdivYBNm;e9a$_4=Tfe2E-WWx^^F$f?8`5FmM&*&*)AKia{fc z6)38!2TV*DxPqFbENqu13qx4Epi7}bBe0Vi7zax+*3+;-@hoh6BU+dZ>hVF690#)c zcd&@(vO*kT!Yo5A^uF|AH_#z5XqUv6`ec}sOY<6Nalpt*KmZ=ZAd8sTkl}td-+SJT zexkeq>_VP6&rbn*VT(Y_JR2JBpyXtBc9wWM%?=kb*$x0);Qf4#D|#6Og`YnYj=HWB zP5+K1h{(*)ECIkh-<#|3SBHy@ts2CBl*sxYGQijaxaw_nb?Sx?+B4az^Ds7`+F&GL z04gb3^8g4yu+FPo0hj=N)CoV@mzKq0xsZNgJi@!*tO4*Ej7a_2Sa(<&VsF8E56T&u ztHGHqFL-M1)4R;?4SzrhrV!Rx-V32>gY!H3?T(*hwD)xFftq~gKO<#Lf~7a_NRmLv zp72H$OJD2?M4kc^Cf54h3_W4|_fWRPLNF9l7q~pN@I(K7@*{tp`clu6(+kkW+ZjG6 zG;TE1o@g~0`EK<_o64go8!o}VU;aSMBQQXW2Q&@@%9X3u`8CS@<({Lm%Xee&H2lc% zeZznCW8RaQasrkJ0z`*MA+X7)^6(2kf=DrOy)hs?0WJuiX1sV80^~cl9t+Fz*W8Qx zBwk$wGI#D+7rar0GDlwK85mIh5YRX~`&rPEd81n@bBLW3VZKN6i2D`P?1bqO6?NRv|begl=7oPhQrBrlF+9u3`Cgu3U zmB?SOb9t&xOM;o!cyXmZd}+mMN-=|9H+7AZ!q_uR5zy zTlb;>WZeZhJr)PY9%&Jvdvv;1V?AM#lM7H|!G@B*>A}=LS&71<{Zh=zj=nxrflFHi zCMN~zYopp4;ryaR2U^zz zs)#)8(Pqihr=N1O#%0*;F!&)&!h`J|ihDvF2-{2tFO+JWu5V;w8^aO=3u-Y(UN{?N zjOf$(Z00JGN>SE7+}kXM(Fru=g7#nh|40TRx!npi zs0XmDodaECVYp1_WyPjyx(WwU9nSv95|xEuyvuy``ok#h+cGjf8$8#qcbje+!oGt| zgGOQ=X#Y{h>jH~2F*Qx`@b&f8$qI5~7yhtu4|Yh2YNANH8jvq21;m4eX8o@K zn-uJPgx)qbF0ON#3G$lM%zO4Gv7e&%oYL1>x^#RaI^lE~5Aa=fvs0gRr2eNb0^nIEu2>?A)x0 z{|*_XPCZV581pT=3sqjkS@gGJ!Kl}GOHCC@15qKfbB7qIPZ>jQrM@GhKR*#E~(A8k3IC;ZD7yRAQ6PR_4`^XCr4JSB4Q&_&)rZ)2?Y zzeQ$@3Gn{Az4gEEAHo?wC2-%Hp4=x5Vy0`CTaQr<9d<3H`xkV93FzX`pnFhrdL#6- z*Qaq(1Z(Ts4NSQ3zbAN&W+#~`F1UM&7!`tpLg|0cR&%HT%;%?m4Y@-0|2bim z;Uu!q9h?w9+5D474##9VT{|VZcD-fHty4zQtPgp*@FX9hHX}cClG6V_e}=COS@@mW z4}GNiQ-49PdubT!@1!$iCZ*mwcpUst%**hgJm^JFodms7_$zNIbh}RtOeV0-h(0K~ z?*6j>V+=HtotoN0AHh~sQSoW!E+YhZvu*eI;)6??C%YxuRhi?N+ixZaLw{!kqlo|= ztlQaLJfUYYkZia-QC%6d0jrriNp`O<1l;zx_fmCdl5=UOu$05THEFC8sS`8pNM@b87m|gD(Wd{CT5TP5)9CZnX_MRRdsu zx0L3;^@lZl|I+v~syt3;(7}kvmfiJ>9H*`km4~C4T1@#rugo}k8RBF^|DS*hwLmvb_`vQc4v0Lh2Fe<>&2y6xKsL?!H^N#;Ga{%DS4o|=9JWMQI{T7N5zz!O{xo%TazsB#cY(B@>y3^BckPfQFQW6y6J z+>|?U%b3%r3E`%Ru3ADiI?xXj--c}oH=$K`M$mi!8`*)u5P#@f!$|08o` zq10?k5o$WR;LZ8oTe-cg7ca_`D!c?W7=2g6O-r;@Iqx)2C3MQxoEXhG0zd z9{MzNP+}`{J~pzEw=839m}`T<(oJV=G1}r&y>3Y2zdtQyX`MJy9v^Y150ot{v(KP~ zf-9CA*=rWy#b-lH?rDnm7STpFqKx%EE@4ca=2)lLm*ja>vRGIGngs7pUcwPeKfTny zj70>O889x;@NWcGCXpHFz*c?gbD{DMBuofE{fRJ-1J;J%3*J!b(brng;q*;;GL2XP!ed`IXx zn8~As+oDtx6ALS4(PO8Mkr#R+XrCB_kQIF*@#N3H{{`i1`Cc4X=qJMDLml%0hLnj3 z2k-)a;6D55&*Pqb8E_;yDL=Pi!$FGcgVE4VduWw_cQ8pvjGQT0BQ|D``rqBT$6!V} z!Q=P{Rp_ihItEpa68f>|6fnF=v9mhqrhPjsfPl!XkdpT_WaQx~fBpLP9a?l~uXG#; zM>YD64NPgF{1wEGHY0(`oTR(o{cr{UXYaJm0tB-f+P0 z%#vg93J7#9ri=h`E7yt+P4tv5J=0dJbW+f7##DuI>Di0ApbpzC-8hOk97<37J7L@4t7MtP3t^dZtel( zn_nZCMqf^&LkvKEL)Xt#-LU$G(-Mkep;VRYhrMQ}LEj36lMi6V-4oz*lAY$XsPaCX z5{8q{U6@U$=b}+u9R=cTJ`AveT8Fk*V9##DU|(+5C8dYwZ+<fgzGy;m4#jS>!i8OO~Y3V_VE6wdiE|1Iq3@}xM(YA!DXK<6PrC|c^8c&bId0nnZ znCL>MWej8GXSz%Kqh57JU)4Fh0XCb`o$4I=a|@&>pnI1tT}sc)G|3<8NO=gxWXqraS6Ec;;zB124|lYSaraR6y;yiQ9V3U*5n z@a5Ce%MYQjbRh0|8q=*HXy4dp8%z3#d(wFdD8+|J>cq|bgs3Y%Td^`nYeW|OC_iiO z@;&Y9kblyH4;nxZe7hi|@EChRJvuq@!`CoO!6)V?-5N4G3RvFd@4HTj9OY>$d;t*j zy{GUfrqTav43(ej+sG++5<9spM~VP`pV9M^em}atl4&l_0;1>tZUaL92T@6)U?NI; zER@Kffs1M>dg4nxLIn6v;Pc5h0jcZ=J^~%e%51bY+nt5Qq^YodIv#Z@{po)ll1x(0 zD~!DJ0eHj+a=N^`yof#K2P+L^4Aej1ws^SJ@=OAXLlypiC|8{jt>3 zF66|9Nn5Caz1ZI>6X>}m*EbfNT-y^vaa!d8PVK9Yn0m2qPs-sy7~D_q)D6yI>0KeU z=SBqIXY;;4!Di`h)QG6pm`#=!Qk_2`!q{`du`po`QXr>r3y>m;I;|NfTRZt>VBT2j z%dukycHbYh6gT19za4yt{kid(Y0Rrx{04RH%BS`c;fE_EmFHy4JFKr-cTo#7qvS?g z1bsGUF;Zhb=E@vg)Iel}p0QF-?;*IUG7rL9RcKw+E})rn16@_zS^ayj@&04T9eRQU zMPQxY4zswjTdSEFMD}+&?6V%$TfDQ8sad?QJ0>( zCvNr|UCsTl?ukmP4;_YIBeqd|uQUkeyM~YRhR&&yV#U^X#Q(%?em5;+>A|$Se2g%R zUQEqr7N;0XRIhJisP@qIN690Vdu(0Shr7@vbyza>oHvvP3Y2T^jpB8@e4DnK4T z!qaj|pdO{?-aL(7ekD|LS_85(8F(eG8pPWKMYcS6<1(05h!}lW7T)7>)2^WGA3XZi zvwpLn(CeKv8d&=Wg*YqF#j1a-rhQ2+>zY{~FM0+vgx0KDG^p%fviv=BWh1=si8mbaE+?~G!JGmPPRf*uqRZ2Q>XL6YDs?2$iRr?B)H}%EUi8`zl^SeQI;1oBuC{_0J zo}XBg7l~Vkq0ywzpFejm=_a@1L1|iPrTUNM6&>y?Gtnm)NE27q_I)>W+iif8=9%?h zLtDV;D^`Hm+YL%!5)k%XdN1nZ#{sVbF@+Q}e2*=>_qxU__r*`nTH8m8L4QVHVuZ=P z>p2x$1FFs3I_11(6_jbR{SNQ7eE9<33Zij z`){J7=jSs^xTI2Kf>p5)$UB~-)rnsV+Gx2>CVH+WRSa_h=-a#TZd(`7iE`t8 zbkGC34q&XM`>*k&t;|_p(hU(UA6RO9Bv`5bpb^8C)fnPkwp;863%IBTV>IZCn|r1B zi`KVQuS)Fh5<+YsNVoI(U`kF*7{--VW$Z<3)6_4EGEY&_kkVN-Dw@9VaU4&sMow3C zR4<*oN{y7gk!$?vs*Qtw#DOnYX&xkRu6Fon5@6jGT?XJ0GX>y?0f*4Y&G_47Zw_ ziJgTSAssdo;6?g;qzY<+BSZ;O%!!_Jp9ghqY8icgy%2nUzm$7nHVSXW`s9=~Jt z{&ak7u>2(Li|=EiJ~ef3!xTGuWg&DaRPZYnHYwj7ivVKd?cc{w!7kBc9@owWaytR>shQA^QSDkG=;)^+o}l6hjvhpbT^-02N_bt!$16esiJK_x;r{qXyL zv|p@7i)A++;Mw&Z4GG=NUzE8`kZWz9T3VxTJ&X0iB!AWDf|;O~H~f{Bkntx^wl{#$ znp2PWh67pj)k4bcV;lGDP!8$DG(nv*xy*&547+F|T9C^rHS+dCTK$i;buMR>cEe{zv$XT}g|wNqEJ9_Y`B+jvqPx*c z{8!Vj>GcFGv84-)DJpSE3viHk3m8B>vkfA&AE?qcWqu158Mw>rXw!WOUZ8*vQm2UA zO`AUI=xg1sj%0HE>(;H92+YCyA^RsBs*4=h$kozP5|&q38eH+`D1!OBiLxUD@rbLR z%_>3ESbhFPEPh^E|4X0c-_Pv#UIqcOexOi5hMv%x_8j6bR2siCk@_t){t=bYY8#N zm{C5l3xNr*m^jUBzQ$jgUyvTFi~Jud8uCu2)(n$=Rw^0NeWJ};$dFr|$DnjCsJ-+m zfC}yZd%wu@O$G(yhE&6a_NBP|*3EOPvvHBl-=6QiA%A+spOvlJu5B%RL!FDeHTdSu z6bzilxY)RCDK6L;OnfY)mkRFq_Zne6F3#~d*ENG%OUx)P zY^JnIj*=GAP8;iG?6=G`jv65~KFRfVf8)=2wMp#`1uCib;#IWI@n#5H)eXc-8_Qw z{Hq5QXE#H*TD67k^dNI%=IuJ(_#HRj5O+PvCFqa2pcfqH(;y0e;QLLk*XZSS;hcdd zNHmgEfKe>QO6l8;yK$fJ-K$Jd!pj6(6kQ1~q^T!rF^Ao?O&4ZBBM@_QI_HT-6Ozs z^DuqQt3@@~ZNT+gkcYa+ae7-hYN1N&#Rd zWuH>hl%n(cN9~uARgwgu5nYLD($wFs%{{mObMGzjogktr@?Jh*Fm{NOE=AL1*Uxo* zmTMQ0>wc-E`pcCS?_*w)HO*3DAUo^d=*+}4UW2{-NoTlS<{#U%lsRn2XVhz}eM(Lm z31Gb~^0dLYou_2>MWdfd%&WIKHNofqY*N2DH~y8fevLA8JXQB+%JHb{=b>j8%J{)$ zWbx|4u5i3@;C`di!-_9{zE;Dvg`|XnI^qIWiZVt;`>Qn(R&aeZ9F45#UzS~cJeOqe z-+ehO)T@>QOB$;?U~X*52q;x3Mw_`7u_0pv%`|f?+LHUQr9!cd4 z^U(!N19hMOyvkb}9zXY~(L`xb?`A=bk3^t8^+!tIsU|(Pwr`Yc4L0-4l1|!Hly}cM zV5{9QRnxf+&hcsG#b}TCW$gGPS4x_Kb6zHwHtu9?xZZr+S^Zuwu4kA_)LH$4B$t}R{VFGHtoYr>&K z?b{3OFGpL8MRV~RTZ!a9-Vmb*c~1LVg*I$<{_xZw=`8F8SWdRvz262(2~Bhd>Z^^>~}3tQ$Z?6t3|T+=1fo zzuNpI%d7Rw}hhcR;(30fs*YB|K{_%=5#PIGN@SU3&55AW<9jB9GmaS)zC3hFos|TjrtoP14 z*uV(?SnHKH{U&w566s5s#cNE?laeZOMhI(mdaf{C2d<%*A~gPIITQoIO}k@Yp&xSB zBPZ>-%cJ=#>fL4%l$v%D>YlLpX&V*%6|tm^Ba&-B;CyQ;dOQR zgl!i>`HOCZo+AZQkeNxQGhdaR0?(+9oR2;Q1ORvTmrk+MQU&KsA~2J`8jzi*3BY|v zaXE{KtOg^mAA5~sGvF-I_nsl^zckzh_5s@52?qANOac%&~octIggPk6pZqEDA zt7}-bu9l!Xd4&I^S>0gF z@tfu!b=x$%IB>42wz>}c;AYTI+Yl(UiDZB+wdbmdf zgE0gO3712jbCTmTDQ4FKEfTg&Rn;iKvaDdp*OsxAdStmymL(^Zh=k+g$cJ~|>NRTU z|A=cQ`6LiUOrUI}?ctnBZ`0c{YfcmTblab)D$|(hzREoWpxBU=6A6@u_t}`b3!{^8VanR9Jqs2zEpFP!7YP-@BA;a$K1?d00j?i~_2GHQU*c%Fd#s62@bR;V?uFpr z)Gw&fZ<&St?(|O;E*~)~s4<|}+0)|#vB3Ge4-=;ZIc;d<{b$!EzXcEw`1EtMiRS+y z8(d27h+bJtJeRQ~Gxen0ogaNnrd|`m%RXmt?onyFAi)F#-_w6l`?DLuJwxw&rHEg= zG5@Z#wC<@f&Evs?5&i_@>R+74;-Tz^=b#KKn>g8CVv`EI^L72NR!OuV+l9xQm(9eL zyzXAvlwK%#IV9=fhAs8zVWu3O%Dh}akDy$O;g`NPy+VQ^yDhZovVgk6@LMzO6{oMXjM-n|HIjl%?5wE3o=AD*4S+j!_a@w#+{egt1 z!S}+9Y0TcJg|L}-a&SUyL4uu%?1mmq08qfZ)hOjzH%z%2tLbOMt4m={l`e+a)bA*Q zvafJxNjMBX%RnuEm~TDAFNnoAAhjT@9S9@Zi@k`x|3)4W=jS1CZR8|m@ndD@yUp6yb>Py$whOUiUth7wN^V^c7-wLvu% z_)0YWSp;b>=4#)GR`By@Je`9hVKRv)?c&_JSI-3Xm>p|9GC(5sum5TK4 zs3{$Dok?WGM&fCU48pRaF&F7W90WJE;@g%r=<5BPnM__)tCtt0fT=Hxtv_dC8;xiF z;t^?g^Y#la*z2rBE7ahdA85T|(4!dvuAG}(yC zo)~#&_ineDrmwMkLm;!fn)UwFwf$!(hvFCb!j@G9$A*)3VrX_5T$)@xe)DyXK4*62 z%$yv?4{{j#`X7B{ZO<48vfzGPw1|x*!R0WhDFy^Y3+$4WQYk-BVCg0aC);(_MNeP19}{xSG7FfE=)8S) zS1o(bX`JcZhteZPU3)SpVaPfKdFW-Ql@sxV_;JpuydD-G$n98Z=MRc|-nap&hzmZ< zsY;?wc+>a~p#xX3HP5m_}5mAnsXdbal@fNhxi{*L`-K?%S8;j>gv+wB#j9x-xB<+!i?X#N@D zX;Q>@T-{egYpd)nJgscQ35XX)FDY>ekJ;rgzzVdk=UN;V>=LuqyCsHAS0QY)Xf6o7 zs1-XCX5y81b`BEHCcn2dvNzLWeXN?ZHLt{;W7~=ySgx-wPDI;igT0uyxAh6w{5F68 z2sC)?`eur8Z;BFRW8eXsjy&w^7oCziVmSROuaAkN3)l0hH0QF?F$if~mb8V|j|A5f z3au@z7|>Kt`s`dUV(KJmI_d{*dFHv5YkByUFM@S5oNbV)i@K#TnVY9@-e`bSC-7PL zOZa<8hCzT3O55_0Ok_r{NGGS@cDH<8JyT5wQ8~DuOYfaH{@psu*~9xnhOnpm4+x*_ z2+}_kvXe(tU&xcksCS<$Z0Z#JOfTR2L8#I?AK9JNagncly7N7rOQ?Tx-M@u~PQQ#x zXtdgt3Kn$1oSDQ=hQNCnEfJ=~umrJGX;|9{*+hl>plAJ$;S=1n=A-v=(C4ZKU&TEr zz4_(C0@cmW{)(p`w%!^~m8IfnGXmXIwphy%s^6=PJeZn*lN_Dc=L8V!ZEnQE;sL3E z>PPDH)#Sb%%!Pykx)*~trIb2A1 z=WgDT;y`GOE0ADZX1Tsv%DT&zNrHbBJ-j z^PR}n#vx?Hbs}4Bw{ESkd1(D`sMcc75h}UCq^L_7I!`yjJarRz$}W0%9za3v=;yC% z@$yppTbOEL4%0#|=aNz~)uR{U_Xju%KpZK7`-)9_A3}({F+GRyOA+Ph2-Yo{dXATck?HUfA zImK@489C^ZKKnV8I@@)t#s(wwheFKVTL8>+hSFrA^q?K{V4QI`x)p$;(ek{uV z@n&eZJOp37ulk?&9GdzG|v!Qt|c*ta5$Hk!IV1q4ab8=L?zf2^OD26kEV6 zR(-N7KR1tjr?!~6YIaWOxkF7cGfOG2AnW4Qj=;RG00@d{cGD?Te{k6}NUhg%_K02g zIm~5zWJRmSOYO<6%V5&Fz_M8$INo%@cdFpq!&BM|@qXXlOfPMHbF*m-=H)%4@L)7T zs6dZyEWx)^+x>mZXc3{G^PQv-{%9`gG>j0x%SVr0 zq4;c-*^RKHqghi+A+R8SKCtUzf$(Bxu~&6Ta1O>~z&@nQ+R^j9s@~M! zt(2+kSL4_#usDz8(j7F4tj*Izt$UpzKZ+Z5Z9D5I1__h9>`n&*DDP+W#xIRd8U*dk zIq9iTXy^}F?OLvNLVVGkyl5)L|>k~WFQ;%b^JIux=oNmX+DCKd0v`&D^qJa$~{7upGt2k-b<)ijh)w@b|Ye(8IuS^i-A;z3ZWDnU27Gobj$~~z_M6(#O z9bk&BTvm=ba0&-g!b75m#lW6B?Z*MJOv!V4^)LmzHVuTw>B(mxwRkPzxzhE-uADUT z^}l|aO5Xb`l$0(+j@N@!xI`zs;E>!o_o6W>ICX)ekmSvq*f1vlN1e^r=eUg*gpNT1 z8#j94XzQG7VI_Cmvd+rdC}fkV{NPMlG4A)x@ZsQG#U1DQP(LFb0mJN97pOic?)U$plnCGy7$()j}O33ruyQt=7rI|hOny>GnbhuqxBY)jsEM*DDVv(fBK zK|wDSu1MnVBcW~ADWDPpG^Z>=!%}tdZQc$-)U>EcK+{4yG*xaga0}y zD!pR&d&PNqC2Jf+g3Mi=b{sdN2&No#tIQ5BQ`4s5d}rib{2mqd?qc^@z(1Yzs4YIx7R>u|;>N<949pSaMPQ}_8s z$5esF8^+Xgd=RlZ9Tiw=D|7IM{Y&3$ek8SrV^pSepWN_`s*m#zo`PuBaJqu;FmB#` zcx!T~>R6vxD#Q$sSk9C;yP`Ca+jg`|`Xs$^+51$Tye*=MX8f~x*PZ|Q5UNpERaO!dS+kd@@8(ADOdFwlL*ehN^EfH+apv0^E%+? z!5vz)7^}hFy|j9N43f2nbuY|FH_j|+#jtY+?J+;E9??$}NxxwezJjqxOr*(STwRPY zp#Q=hyeY6lV6Nj2QvpF68_=r>1HPAey+N5`qUJqb#Zj!gFb;e~CDG@sYO|aXteexJ z=SvGOb78%_u9nH4c}Y82c(!=WH#VtmUSbEKVu_0PyF50QnwYT%(v&cw4ss3gp&ssz zLlGn0QrQ`cb(mGHFkRxAce*&W*G4;vH2oWv4sxBy^_sg?<*cx$-AF>+(Q?>a(#qS- zCi<<>i@SExUS-)=U!%Ke2q7*_hiPiJ>na$Mu!j`k3_Fn6XzJ4;#sJ9@#_XNcs+3xR^iDyQ6U1qKb<+W+t$9v_s zXapIhH66XtzPp=FpI@ZRl2_X({a_okFL(^uzSQ($Cb&M@ z(zXUnY(o&y1$J1-g`^A->a^ygC@`#+Jv&3&`>mZ`9hLB^Zg&rfB@HT9Y&fm-vNX)R$RHfns`@~>Cc%c9Y z^50}e2I9VVtIhrAA+Zq6$rACNuV<-F>NC^8irHiTl9B{s0SnF@U zcWr(9n-U<+z*tf}kLz}``zm?fNx_7G;Z+(EzikoNatDrr$#-^pJ|b^at7-24bYTT46m z2+wnd6Z*yQRi(ffF~FXdGOnF^O|LFJWwPIY@^FVw5u@6@vzEZHvYZ<(A15 zdvTSUzf^UF3=SmZE}2ASh3is{K(52fLX7S?y|5`$9Gyb8{y`CiSy3MM%kJQxgHd6s z&8vTt_iaQriIV1w#Ig>VI_Eokn-MsUT$1Ml;dm7`=RHKuck1D^{*-W-o)>q-v>Hz3 zs^m;=PDUv%28o!(u!P@sk`{8Bk7WNgwekY!TSS4h^Fr=R`NZ1VPRg^|AO^Kobo`fAXziN%qN=a#n;lLs4u9s$Gn=CX zJw-G{Ix4vGD16iHHoTV=El-xQ7I9EIads9+jc%sd|l zxoORLPAFK?7~bg>r81uG5^D4o;wJL!r_!OusH1|>##S}auhoBQTv>0IGk*%cnF1VI zM6m9w0?VoTiXrc(7EaAKEu*pa7}6<$3~3N2<4+>pY`#+T6nq&olN8b2mf$LBQm1b$ z3-?W?;ulW5h~t*9Ea~qvaOerWrfhW~@U2nnt(HmHetAk>;9A|aeluuBs@T4v1qb6; z&S_q`<20JoOFh{F@L|m{yyo%qGH}vY{Hb^`>1`72Q?gKQwRbJ97h<{fOz&&7iWb%W z#kmtBjYujFV5BBsq6Gt=a|-YwzdYlh;7kWRxYsgC=2|ObMHgU9l|+duR68Zx$}U$| z>+H)4z-KqWm!^7tflTU49U>?xnJ_}=n_|Qdhu$gYBs`z~()EzlCvzXCPA?|+>RWKg8$cP&$~^T_d$dit9Q^*$g;UiowB_Vz3>XNp5uDI8J|L`=W6`A@M9lvKS?yFbvsNz@UU;d{A0H;Y0^f0XDZ);FcuW{E#S zhCGyM?pJ^J+tZgAbNgxtg$EBQd=aIzpz*iVoP&cy<$YO@*O!7*c8HXQ4|A1;PcIKg z?MP{~_Seh^00=k~D@OvO6V-#?i@P_ZN=1EaKemG$*LEbOlAwPFiVSo};-iiNK?+a< z1-bhC(X`h2k^^28IeoV_`T@rfs~+JZn;XSt&i)%?W+0fY^mK_#b|>yXufLByQk6aO z^Wx>s*`Zsp!l+s(|N06xMA#09R!|`kjoKfqmjkQ_3hbZ|2@8z~RGpl1f9AjJTpoIY z{_NHyJRwd)Pp?8Cq~vc)%zB0YDWUy-8kjhPfIU;H)z@{igs zIr~~@S#lzQu00vfP?Vu^5OWZumd)FRhsD7Dfg*_@kcTqUO?G@MNd&nIklq+sS+T)N zlU=^^+w7}U00#psfP1t=e$w?P7q?c&WWi5*ZZ){z5`T~ayyv~x*r$f;pGRG;P+Zvw z5F4P7=l_lX>PYpfh|j*Lw9iW>KDeli%S)0zcjfnM-F8}g^`m1Dh$bo@nz!{ys98&j zHqe-+K%A|*J%ZiY-z3pR=vwUdTdW_zZP)&PKmG5xZdW<~S4S5YZiUC$`T5PDWQhQR zGjN-jL0|2Ma92fAi06Rxp3waQ?$wjJO#JrUH zlF9d$><_wepl#npiv8kY=XzkJC$yHjZ$+DOeBWP;X;p{PeW*8t?!SLQyq} zn<21vV*a@&DIkCnps1JDzJG-8Hdy5=4TRMwRN{~Oh;9Nz{7BjYfC40SO7fnfO^%DH zfVY|k{C_h5YGyoyx4uQ)ZUA|H6p%p{M#`9hA~g8YDzKxK+y`)jyTHUF03QU=ND{rq z!T>p7;4?-LA%I-pq}+oa1P|ZfJ6gjfSubAv)Zi#XJ!cTOb{b?%FHMen!>?Vy<^T+h zrY)ezLZJ%M8dSxdvmS`5g!;vVZh6>noLL2=18C0ufm+NCq*PvzOMNhGN}LZJX^oDe zy%!0zeH9y9TUwx<2Qh-iR|#OJ;nkZTPsrt6Y(6F^2oEX2W$@RCdq)_sX8>B1?gTwt z_MkojKfT+Yh$%hQCn<;G*3u&q_Di}bc+B_fpneXNe#cInc$g&Z^D_&2d04{>yNiK6 zX$lVysO9~WwjYR4yNNCaDZk;6op}6NXXm`1s{kn33sR!g!a@TY9u;5TH|lzNY>0%a z*Gj%%Xn21WIszP*?Vn%fJ$_t!^UKxSvBi-br&SR*w0k;H+KUb|1#E?6cy%q%k5pV; z3mBDXFjLTX#6@7beOcMrmRtgL=7)rYbdFn84q6QZik}Dgn6qh|Yh%EXqQcC+T?C8* zr5~b(8a5G&?toJ48)&+(d2^iUnO{x?Oy(5mMVO)EgD0IIh_G4Ue(Q)un#09c$AvK@ zKudahbCf+fefciCqzfU(RyBg12leO)b%H|TJ)lZog2&lPCNshcx&N~iG?B0}_&Y(5 zw+n-_GNs`ueD$gkaAxB$NLd+}nZI{ENPV1HHH`_RO}Z-;Kn$1!X_MUZxJ6&6pM^CA zfFNFsdUNWUtARiq4BEq_dGaZ@Uo|^+6MD1GAsVfj+-;DO002q)c#!k${I)1XwYZoZ z7_%u@u?>0V6^Jxi3959-YYUH*AA=yPVsqis$B(r(r_dL|1I4C4-!f|0xq<a+v^4sTe#_~U5<^X63&8}9qfRr5f+UI6%S#9%%WOADB(8h}B3 z17@GPE#TaLbzDPN_hjl#1HinKSOi?DKw#~`YMP4eaK8NerX~5GnZD?{_oX%rGfJZN z_e)*L&(59|P1}%h0Q8zksSD4T?}|Clsx^k37 z$Ss$XeOWOs+YI&OKbx14q6KLX5y=!(L}x1Vo;}0XtgVd2*nSWIiWN|cBweS13q5Z6Yp=&iRFs%E!?H12n46*<>SjBfUucIr>qNTq6yF^$SCK6PvX`a zae3_i3#xgDS#oP@{ft>DOk;nb02^MoKwe!pU%)E&!4a|BZ7?pFTrh;H~va(S1y<|l6{2Wq@ z0=KyVX_#5xf_j()@;>z_<6Qhqd-v|8H|M|k^#u9^+`|g?_4blrlMRiZkS>hi0vsU#>4ebG zJD8fACxYlWeih7$pi{i0YbxppBPz8Cy$66Tv|E02BKDM9AS>4Y??xg7avmu1X9~nJ z>;XtR6EceGT3XDobl4&zBdhZ8p}>vxgFaonM~|w3dX_vn4(PIUH$b7Xfm?uf(kTGi zvEg%#WT2S^ZexyoD$zd&b1aAvdVU1JjNbk?bd6yGg{&@H_1)uqYU1%ah#CzGIUH`x z(#&hr6B)ysTgw9mk3o}r5@cV9%<2l{>2TBz1GuSLaH%k#G1|g{!HB?$VFPB=UU-D5 z&@)E2hqIiP{wR^S-yzx(P0?3&OW2nl}yiGomA4BznaN^FbcUDx-Barcl@K@{6BriZKdV^=!8ekpY@Oe zNSUQ^ASp{3Y7b*9UP;zxqSl6UW6s68c;hDUzOM1@j1%7-?e0(@GX4wIKotgj~$XA51v$eJAN<9k> z%Z^(h!vtycm7&+PASpb85b&YC`2a};KB20orx-{k!$D$=U}QhLq!07D9e`RnPX}0v zfV5$k@;HV$2y8XP9C34VGkY*Fzl?YyfQ{w{-y{wAsNnyj#tTbegqxo(fJ213lPP8x z@PKq2rfA*1tCf?>E8zKCfO^!AM2-ji*~!c8T!=u8dWgWz=~~YqaZpNX$ad{g^W9o| z(J7!_`9Z<8cb-}lH!ccrPBTm1AQ}fRMh$Kmo{P-79OxyHT(I|}FEF#w{KD?$j)1ll z7_h@gr~J0?S7!+fezxi0@5QYs=?;zu5if=job`zo|{LKGW&`>f1yEK`gc%%XD-e*tda3Z5%^ z1JslN_>L0@NGbx8PA52EN%eY{hBy zRlB)mNJzbou)5G_>JIe$p@kv*-Z*@Sm&hDkx}=BaO%?tG4o^L_7C~B6=)p7rHmeyx z>=+Dc@&r3p=$jiWO8EoR@D541_uEG;eo literal 0 HcmV?d00001 diff --git a/perun/ministral_3b_aya/training/validation_loss.csv b/perun/ministral_3b_aya/training/validation_loss.csv new file mode 100644 index 0000000..532912e --- /dev/null +++ b/perun/ministral_3b_aya/training/validation_loss.csv @@ -0,0 +1,111 @@ +step,epoch,eval_loss +1000,0.009168928335656128,2.360072612762451 +2000,0.018337856671312255,2.0450692176818848 +3000,0.027506785006968387,1.8085086345672607 +4000,0.03667571334262451,1.6807985305786133 +5000,0.045844641678280645,1.6042695045471191 +6000,0.05501357001393677,1.5622832775115967 +7000,0.0641824983495929,1.5304133892059326 +8000,0.07335142668524902,1.5210760831832886 +9000,0.08252035502090516,1.5131747722625732 +10000,0.09168928335656129,1.5016822814941406 +11000,0.10085821169221741,1.4817888736724854 +12000,0.11002714002787355,1.4736380577087402 +13000,0.11919606836352967,1.4612903594970703 +14000,0.1283649966991858,1.4510605335235596 +15000,0.13753392503484194,1.4477511644363403 +16000,0.14670285337049804,1.440198540687561 +17000,0.15587178170615418,1.4364843368530273 +18000,0.1650407100418103,1.430030345916748 +19000,0.17420963837746645,1.4217251539230347 +20000,0.18337856671312258,1.4251708984375 +21000,0.1925474950487787,1.4137701988220215 +22000,0.20171642338443482,1.414902687072754 +23000,0.21088535172009096,1.4114439487457275 +24000,0.2200542800557471,1.4006237983703613 +25000,0.2292232083914032,1.396726369857788 +26000,0.23839213672705933,1.3982290029525757 +27000,0.24756106506271547,1.3934869766235352 +28000,0.2567299933983716,1.3892569541931152 +29000,0.26589892173402774,1.3871415853500366 +30000,0.2750678500696839,1.3804394006729126 +31000,0.28423677840534,1.3778815269470215 +32000,0.2934057067409961,1.369511365890503 +33000,0.3025746350766522,1.3662126064300537 +34000,0.31174356341230836,1.362453579902649 +35000,0.3209124917479645,1.3640830516815186 +36000,0.3300814200836206,1.3563292026519775 +37000,0.33925034841927676,1.350153923034668 +38000,0.3484192767549329,1.3465571403503418 +39000,0.35758820509058903,1.3438115119934082 +40000,0.36675713342624516,1.3429770469665527 +41000,0.37592606176190124,1.3332430124282837 +42000,0.3850949900975574,1.334341049194336 +43000,0.3942639184332135,1.3271080255508423 +44000,0.40343284676886965,1.322904348373413 +45000,0.4126017751045258,1.3139293193817139 +46000,0.4217707034401819,1.3120628595352173 +47000,0.43093963177583805,1.3108934164047241 +48000,0.4401085601114942,1.3062814474105835 +49000,0.4492774884471503,1.3023930788040161 +50000,0.4584464167828064,1.2967934608459473 +51000,0.46761534511846253,1.2952461242675781 +52000,0.47678427345411867,1.288960576057434 +53000,0.4859532017897748,1.2882647514343262 +54000,0.49512213012543094,1.2822253704071045 +55000,0.504291058461087,1.275566577911377 +56000,0.5134599867967432,1.2699377536773682 +57000,0.5226289151323993,1.268863320350647 +58000,0.5317978434680555,1.2632348537445068 +59000,0.5409667718037116,1.2605316638946533 +60000,0.5501357001393677,1.258490800857544 +61000,0.5593046284750238,1.2527873516082764 +62000,0.56847355681068,1.249812364578247 +63000,0.5776424851463361,1.2420779466629028 +64000,0.5868114134819922,1.2373592853546143 +65000,0.5959803418176484,1.2303643226623535 +66000,0.6051492701533044,1.230921983718872 +67000,0.6143181984889606,1.2246050834655762 +68000,0.6234871268246167,1.224005103111267 +69000,0.6326560551602729,1.2206971645355225 +70000,0.641824983495929,1.2165570259094238 +71000,0.6509939118315852,1.2117177248001099 +72000,0.6601628401672412,1.2068102359771729 +73000,0.6693317685028973,1.2016609907150269 +74000,0.6785006968385535,1.1967077255249023 +75000,0.6876696251742096,1.1926217079162598 +76000,0.6968385535098658,1.1919411420822144 +77000,0.7060074818455219,1.1873736381530762 +78000,0.7151764101811781,1.1825459003448486 +79000,0.7243453385168341,1.1790175437927246 +80000,0.7335142668524903,1.1748923063278198 +81000,0.7426831951881464,1.1718367338180542 +82000,0.7518521235238025,1.1701050996780396 +83000,0.7610210518594587,1.1669533252716064 +84000,0.7701899801951148,1.163935899734497 +85000,0.7793589085307709,1.1622910499572754 +86000,0.788527836866427,1.159493088722229 +87000,0.7976967652020832,1.1554956436157227 +88000,0.8068656935377393,1.1514334678649902 +89000,0.8160346218733955,1.149964690208435 +90000,0.8252035502090516,1.1484603881835938 +91000,0.8343724785447076,1.1455662250518799 +92000,0.8435414068803638,1.145362377166748 +93000,0.8527103352160199,1.1426365375518799 +94000,0.8618792635516761,1.1414161920547485 +95000,0.8710481918873322,1.139862060546875 +96000,0.8802171202229884,1.1378090381622314 +97000,0.8893860485586444,1.13651442527771 +98000,0.8985549768943006,1.1353960037231445 +99000,0.9077239052299567,1.1344630718231201 +100000,0.9168928335656128,1.1332764625549316 +101000,0.926061761901269,1.1325769424438477 +102000,0.9352306902369251,1.132089614868164 +103000,0.9443996185725813,1.1317222118377686 +104000,0.9535685469082373,1.1315722465515137 +105000,0.9627374752438935,1.1312942504882812 +106000,0.9719064035795496,1.1310818195343018 +107000,0.9810753319152058,1.131012201309204 +108000,0.9902442602508619,1.130906343460083 +109000,0.999413188586518,1.1308903694152832 +109064,1.0,1.1309707164764404 diff --git a/perun/scripts/evaluate.py b/perun/scripts/evaluate.py index 52bbc4d..07a9d7f 100644 --- a/perun/scripts/evaluate.py +++ b/perun/scripts/evaluate.py @@ -11,7 +11,7 @@ from pathlib import Path import subprocess import sys import unicodedata -from train import ROOT, read_config, write_config, verify_data, verify_length_filter, run_logged +from train import ROOT, read_config, write_config, verify_data, run_logged def normalized_tokens(text: str) -> list[str]: text = unicodedata.normalize("NFKC", text).casefold() @@ -231,8 +231,7 @@ def main() -> int: parser.error("--max-new-tokens must be positive.") config = read_config(args.config.resolve()) data_dir = Path(config["dataset_dir"]) - manifest = verify_data(data_dir, (args.split,)) - verify_length_filter(manifest, config) + verify_data(data_dir, (args.split,)) adapter = args.adapter.resolve() if args.adapter else Path(config["output_dir"]) if not (adapter / "adapter_config.json").is_file(): raise ValueError(f"Missing trained adapter: {adapter}") diff --git a/perun/scripts/prepare_data.py b/perun/scripts/prepare_data.py index c292fb8..0ed3436 100644 --- a/perun/scripts/prepare_data.py +++ b/perun/scripts/prepare_data.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Prepare Slovak Aya train/eval/test data for LLaMA-Factory.""" +"""Prepare one shared Slovak Aya train/eval/test dataset for LLaMA-Factory.""" from __future__ import annotations @@ -7,7 +7,6 @@ import argparse from collections import Counter from datetime import datetime, timezone import hashlib -from itertools import islice import json from pathlib import Path import sqlite3 @@ -58,18 +57,32 @@ def local_rows(path: Path) -> Iterator[Any]: raise ValueError(f"Invalid JSON at {path}:{line_number}: {error}") from error +def parquet_rows(parquet: Any) -> Iterator[dict[str, Any]]: + with parquet: + for batch in parquet.iter_batches(batch_size=2048): + yield from batch.to_pylist() + + def open_source(args: argparse.Namespace) -> tuple[Iterator[Any], dict[str, Any]]: - if args.local_jsonl: - path = args.local_jsonl.expanduser().resolve(strict=True) + local_path = args.local_parquet or args.local_jsonl + if local_path: + path = local_path.expanduser().resolve(strict=True) if not path.is_file(): raise ValueError(f"Local source is not a file: {path}") - return local_rows(path), { - "kind": "local_jsonl", + source = { + "kind": "local_parquet" if args.local_parquet else "local_jsonl", "path": str(path), "sha256": file_sha256(path), "bytes": path.stat().st_size, "revision": None, } + if args.local_parquet: + try: + from pyarrow.parquet import ParquetFile + except ImportError as error: + raise RuntimeError("Install pyarrow to read --local-parquet input.") from error + return parquet_rows(ParquetFile(path)), source + return local_rows(path), source try: from datasets import load_dataset @@ -148,94 +161,15 @@ def create_database(path: Path) -> sqlite3.Connection: return connection -SYSTEM_TEXT = "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n" -USER_TEXT = "### Instruction:\n{}\n\n### Response:\n" - - -class SFTLengthFilter: - """Keep a whole pair only when its exact Alpaca length fits both model configs.""" - def __init__(self, config_dir: Path): - import yaml - from transformers import AutoTokenizer - self.profiles = {} - self.excluded_by_model = Counter() - self.max_accepted_tokens = Counter() - for name in ("mistral", "olmo"): - settings = yaml.safe_load((config_dir / f"{name}.yaml").read_text(encoding="utf-8-sig")) - if settings.get("template") != "alpaca": - raise ValueError("Length filtering currently requires the Alpaca template for both models.") - limit = int(settings["cutoff_len"]) - if limit < 1: - raise ValueError("cutoff_len must be positive.") - tokenizer = AutoTokenizer.from_pretrained(settings["model_name_or_path"], - revision=settings["model_revision"], use_fast=True, fix_mistral_regex=True, - model_max_length=10**9) - backend = tokenizer.backend_tokenizer - backend.no_truncation() - backend.no_padding() - if tokenizer.eos_token_id is None: - raise ValueError(f"Missing EOS token: {name}") - self.profiles[name] = { - "model_name_or_path": settings["model_name_or_path"], - "model_revision": settings["model_revision"], "template": "alpaca", - "cutoff_len": limit, "tokenizer_class": type(tokenizer).__name__, - "system_tokens": len(tokenizer.encode(SYSTEM_TEXT, add_special_tokens=False)), - "ending_tokens": 1 + len(tokenizer.encode("\n\n", add_special_tokens=False)), - "backend": backend, - } - - def lengths(self, examples: list[dict[str, str]]) -> list[dict[str, int]]: - prompts = [USER_TEXT.format("\n".join(part for part in (row["instruction"], row["input"]) if part)) - for row in examples] - responses = [row["output"] for row in examples] - result = [{} for _ in examples] - for name, profile in self.profiles.items(): - backend = profile["backend"] - encode = getattr(backend, "encode_batch_fast", backend.encode_batch) - inputs = encode(prompts, add_special_tokens=False) - targets = encode(responses, add_special_tokens=False) - for item, source, target in zip(result, inputs, targets): - item[name] = profile["system_tokens"] + len(source) + len(target) + profile["ending_tokens"] - return result - - def over_limit(self, lengths: dict[str, int]) -> list[str]: - return [name for name, length in lengths.items() if length > self.profiles[name]["cutoff_len"]] - - def report(self) -> dict: - return {"enabled": True, - "models": {name: {key: value for key, value in profile.items() if key != "backend"} - for name, profile in self.profiles.items()}, - "excluded_by_model": {name: self.excluded_by_model[name] for name in self.profiles}, - "max_accepted_tokens_before_pair_dedup": {name: self.max_accepted_tokens[name] for name in self.profiles}, - "excluded_log": "excluded_long.jsonl"} - - -def ingest(connection: sqlite3.Connection, rows: Iterator[Any], seed: int, - length_filter: SFTLengthFilter, excluded) -> Counter: +def ingest(connection: sqlite3.Connection, rows: Iterator[Any], seed: int) -> Counter: counts: Counter = Counter() - indexed = enumerate(rows) - next_progress = 10000 - while batch := list(islice(indexed, 512)): - candidates = [] - for source_index, row in batch: - counts["source_rows"] += 1 - converted = convert_row(row) - if converted is None: - counts["invalid_or_empty_rows"] += 1 - else: - candidates.append((source_index, *converted)) - measured = length_filter.lengths([data for _, data, _ in candidates]) - for (source_index, data, metadata), lengths in zip(candidates, measured): - over = length_filter.over_limit(lengths) - if over: - counts["filtered_long_rows"] += 1 - length_filter.excluded_by_model.update(over) - excluded.write(json.dumps({"source_row_index": source_index, "source_id": metadata.get("id"), - "dataset_name": metadata.get("dataset_name"), "task_type": metadata.get("task_type"), - "token_lengths": lengths, "over_limit_models": over}, ensure_ascii=False) + "\n") - continue - for name, length in lengths.items(): - length_filter.max_accepted_tokens[name] = max(length_filter.max_accepted_tokens[name], length) + for source_index, row in enumerate(rows): + counts["source_rows"] += 1 + converted = convert_row(row) + if converted is None: + counts["invalid_or_empty_rows"] += 1 + else: + data, metadata = converted prompt_key = [normalized(data["instruction"], fold_case=True), normalized(data["input"], fold_case=True)] group_id = digest_json(prompt_key) stable_id = digest_json([prompt_key, normalized(data["output"])]) @@ -255,15 +189,14 @@ def ingest(connection: sqlite3.Connection, rows: Iterator[Any], seed: int, """INSERT INTO prompt_groups (group_id, order_key, size) VALUES (?, ?, 1) ON CONFLICT(group_id) DO UPDATE SET size = size + 1""", (group_id, digest_json([seed, group_id]))) - if counts["source_rows"] >= next_progress: + if counts["source_rows"] % 10000 == 0: connection.commit() - print(f"Read {counts['source_rows']:,}; filtered long {counts['filtered_long_rows']:,}; retained {counts['retained_rows']:,}", flush=True) - next_progress += 10000 + print(f"Read {counts['source_rows']:,}; retained {counts['retained_rows']:,}", flush=True) connection.commit() - for key in ("source_rows", "invalid_or_empty_rows", "filtered_long_rows", "duplicate_pairs_removed", "retained_rows"): + for key in ("source_rows", "invalid_or_empty_rows", "duplicate_pairs_removed", "retained_rows"): counts.setdefault(key, 0) if not counts["retained_rows"]: - raise ValueError("The source contains no usable examples after validation and length filtering.") + raise ValueError("The source contains no usable examples after validation.") connection.execute("CREATE INDEX prompt_group_order ON prompt_groups(order_key, group_id)") connection.execute("CREATE INDEX records_group ON records(group_id)") connection.commit() @@ -348,7 +281,9 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--revision", help="Hugging Face revision; resolved to an immutable commit before loading.") - parser.add_argument("--local-jsonl", type=Path, help="Offline JSONL input in Aya or Alpaca format.") + local_input = parser.add_mutually_exclusive_group() + local_input.add_argument("--local-jsonl", type=Path, help="Offline JSONL input in Aya or Alpaca format.") + local_input.add_argument("--local-parquet", type=Path, help="Offline Parquet input in Aya or Alpaca format.") parser.add_argument("--eval-size", type=int, default=1000, help="Maximum number of eval records (default: 1000).") parser.add_argument("--test-size", type=int, default=1000, help="Maximum number of test records (default: 1000).") parser.add_argument("--seed", type=int, default=42) @@ -356,7 +291,7 @@ def main() -> int: args = parser.parse_args() if min(args.eval_size, args.test_size) < 0: parser.error("Holdout sizes must be nonnegative.") - if args.local_jsonl and args.revision: + if (args.local_jsonl or args.local_parquet) and args.revision: parser.error("--revision applies only to Hugging Face input.") output = args.output_dir.expanduser().resolve() if output.exists() and (not output.is_dir() or any(output.iterdir())): @@ -364,21 +299,15 @@ def main() -> int: # Resolve source before creating output, so authentication or dependency # errors do not produce a misleading empty data directory. rows, source = open_source(args) - length_filter = SFTLengthFilter(Path(__file__).resolve().parents[1] / "configs") output.mkdir(parents=True, exist_ok=True) database_path = output / "preparation.sqlite3" connection = create_database(database_path) try: - excluded_path = output / "excluded_long.jsonl" - with excluded_path.open("x", encoding="utf-8", newline="\n") as excluded: - counts = ingest(connection, rows, args.seed, length_filter, excluded) + counts = ingest(connection, rows, args.seed) splits = assign_splits(connection, args.eval_size, args.test_size) if sum(item["rows"] for item in splits.values()) != counts["retained_rows"]: raise RuntimeError("Split counts do not sum to the retained record count.") files = emit_splits(connection, output, splits) - files[excluded_path.name] = {"sha256": file_sha256(excluded_path), - "bytes": excluded_path.stat().st_size, - "rows": counts["filtered_long_rows"]} registration = { f"aya_{split}": { "file_name": f"{split}.jsonl", "formatting": "alpaca", @@ -404,7 +333,6 @@ def main() -> int: "source": source, "seed": args.seed, "counts": dict(counts), - "length_filter": length_filter.report(), "requested_holdout_max_rows": {"eval": args.eval_size, "test": args.test_size}, "splits": splits, "source_breakdown": breakdown, diff --git a/perun/scripts/train.py b/perun/scripts/train.py index 25630a6..3f2a6ed 100644 --- a/perun/scripts/train.py +++ b/perun/scripts/train.py @@ -70,18 +70,6 @@ def verify_data(data_dir: Path, splits: tuple[str, ...]) -> dict: return manifest -def verify_length_filter(manifest: dict, config: dict) -> None: - report = manifest.get("length_filter", {}) - if report.get("enabled") is not True: - raise ValueError("Prepared data has no shared length filter. Rerun prepare_data.py into data/aya_slovak_4096.") - candidates = [profile for profile in report.get("models", {}).values() - if profile.get("model_name_or_path") == config["model_name_or_path"] - and profile.get("model_revision") == config["model_revision"] - and profile.get("template") == config["template"]] - if not candidates or not any(0 < int(profile["cutoff_len"]) <= int(config["cutoff_len"]) for profile in candidates): - raise ValueError("The data was not length-filtered for this model, revision, template and context limit.") - - def run_logged(command: list[str], log: Path, env: dict | None = None) -> None: print("Running:", " ".join(command), flush=True) with log.open("a", encoding="utf-8") as handle: @@ -105,25 +93,27 @@ def context_settings(settings: dict, original_config) -> dict: requested = int(settings["cutoff_len"]) native = int(original_config.max_position_embeddings) if requested > native or settings.get("rope_scaling"): - raise ValueError("The shared 4096-token scenario uses the original model contexts without RoPE extension.") + raise ValueError(f"Use cutoff_len <= {native} within the model's native context and omit rope_scaling.") return {"requested_length": requested, "base_positions": native} -def reject_truncation(source_len: int, target_len: int, cutoff_len: int) -> None: - if source_len + target_len > cutoff_len: - raise ValueError(f"A complete SFT example requires {source_len + target_len} tokens, " - f"but only {cutoff_len} are available. Stopping instead of truncating input or answer.") +def reserve_generation_space() -> None: + # Reserve space before batching so the trainer sees the same prompt length + # as model.generate when it masks the prompt in the generated sequence. + from llamafactory.data.processor import unsupervised + original = unsupervised.infer_seqlen -def require_complete_examples() -> None: - from llamafactory.data.processor import supervised, unsupervised - for module in (supervised, unsupervised): - original = module.infer_seqlen + @functools.wraps(original) + def with_generation_space(source_len: int, target_len: int, cutoff_len: int): + source, target = original(source_len, target_len, cutoff_len) + source = min(source, cutoff_len - 1) + if source < source_len or target < target_len: + print(f"Generation preprocessing truncated tokens: input {source_len} -> {source}, " + f"label {target_len} -> {target}, context {cutoff_len}.", flush=True) + return source, target - def checked(source_len, target_len, cutoff_len, _original=original): - reject_truncation(source_len, target_len, cutoff_len) - return _original(source_len, target_len, cutoff_len) - module.infer_seqlen = checked + unsupervised.infer_seqlen = with_generation_space def bound_generation_context(model, context_length: int, default_max_new_tokens: int) -> None: @@ -141,7 +131,7 @@ def bound_generation_context(model, context_length: int, default_max_new_tokens: prompt_length = int(inputs.shape[-1]) available = context_length - prompt_length if available < 1: - raise ValueError("The complete input leaves no room for an answer inside the context limit.") + raise ValueError("The preprocessed input leaves no room for an answer inside the context limit.") requested = kwargs.get("max_new_tokens") requested = int(default_max_new_tokens if requested is None else requested) if requested < 1: @@ -149,7 +139,7 @@ def bound_generation_context(model, context_length: int, default_max_new_tokens: kwargs["max_new_tokens"] = min(requested, available) if requested > available: print(f"Generation limit: {available} new tokens after {prompt_length} input tokens " - f"(context {context_length}); the complete input is preserved.", flush=True) + f"(context {context_length}).", flush=True) return original_generate(*args, **kwargs) model.generate = MethodType(bounded_generate, model) @@ -243,7 +233,10 @@ def worker(config_path: Path) -> None: context = context_settings(config, original_config) print("Model context:", json.dumps(context), flush=True) loaded = require_unsloth_backend(config, context) - require_complete_examples() + if config.get("do_predict") and config.get("predict_with_generate"): + if context["requested_length"] < 2: + raise ValueError("Generation requires cutoff_len >= 2.") + reserve_generation_space() from llamafactory.train.tuner import run_exp run_exp(config) if config.get("do_train") and local_rank == 0: @@ -252,15 +245,24 @@ def worker(config_path: Path) -> None: def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("config", type=Path, help="configs/mistral.yaml or configs/olmo.yaml") + parser.add_argument("config", type=Path, help="LLaMA-Factory model YAML.") parser.add_argument("--run-dir", type=Path, help="New run directory; adapter is saved inside adapter/.") parser.add_argument("--resume", type=Path, help="Checkpoint inside the original run's adapter/ directory.") + parser.add_argument("--pilot-steps", type=int, help="Limit a pilot to this many optimizer steps; requires --run-dir.") + parser.add_argument("--pilot-samples", type=int, + help="Maximum rows per dataset in a pilot (default: 2048 or all eval rows, whichever is larger).") parser.add_argument("--dry-run", action="store_true", help="Validate data/config; do not import GPU libraries.") parser.add_argument("--worker", action="store_true", help=argparse.SUPPRESS) args = parser.parse_args() if args.worker: worker(args.config.resolve()) return 0 + if args.pilot_steps is None and args.pilot_samples is not None: + parser.error("--pilot-samples requires --pilot-steps.") + if args.pilot_steps is not None and (args.pilot_steps < 1 or args.run_dir is None): + parser.error("--pilot-steps must be positive and requires --run-dir.") + if args.pilot_samples is not None and args.pilot_samples < 1: + parser.error("--pilot-samples must be positive.") config = read_config(args.config.resolve()) if config.get("stage") != "sft" or config.get("finetuning_type") != "lora": raise ValueError("Expected an SFT / LoRA training configuration.") @@ -273,7 +275,17 @@ def main() -> int: batch = GPUS * config["per_device_train_batch_size"] * config["gradient_accumulation_steps"] data_dir = Path(config["dataset_dir"]) manifest = verify_data(data_dir, ("train", "eval")) - verify_length_filter(manifest, config) + train_rows = manifest["splits"]["train"]["rows"] + eval_rows = manifest["splits"]["eval"]["rows"] + if args.pilot_steps is not None: + # LLaMA-Factory limits both splits before tokenization. Keep the complete + # validation split, even if the caller prepared more than 2048 eval rows. + sample_limit = args.pilot_samples or max(2048, eval_rows) + if sample_limit < eval_rows: + parser.error(f"--pilot-samples must be at least {eval_rows} to retain the full eval split.") + config.update(max_steps=args.pilot_steps, max_samples=sample_limit, + eval_strategy="steps", save_strategy="steps", + eval_steps=args.pilot_steps, save_steps=args.pilot_steps) if args.run_dir: config["output_dir"] = str(args.run_dir.resolve() / "adapter") run_dir = Path(config["output_dir"]).parent @@ -288,11 +300,13 @@ def main() -> int: raise ValueError("Resume requires the original configuration and prepared data.") elif run_dir.exists() and any(run_dir.iterdir()): raise ValueError(f"Run directory is not empty; choose --run-dir for a new run: {run_dir}") - print(json.dumps({"gpus_per_training_run": GPUS, "effective_batch": batch, - "cutoff_len": config["cutoff_len"], "bf16": True, "use_unsloth": True, - "train_rows": manifest["splits"]["train"]["rows"], - "eval_rows": manifest["splits"]["eval"]["rows"], - "adapter_output": config["output_dir"]}, ensure_ascii=False, indent=2)) + summary = {"gpus_per_training_run": GPUS, "effective_batch": batch, + "cutoff_len": config["cutoff_len"], "bf16": True, "use_unsloth": True, + "train_rows": train_rows, "eval_rows": eval_rows, + "adapter_output": config["output_dir"]} + if args.pilot_steps is not None: + summary.update(pilot_steps=args.pilot_steps, pilot_train_rows=min(train_rows, config["max_samples"])) + print(json.dumps(summary, ensure_ascii=False, indent=2)) if args.dry_run: print("Configuration/data checks passed. No training started.") return 0