From 581ae22dbb3d2444d19d49849ca210c741be3286 Mon Sep 17 00:00:00 2001 From: Jakub Date: Sun, 4 Oct 2026 23:50:18 +0200 Subject: [PATCH] lmeval --- README.md | Bin 28578 -> 0 bytes README_lmeval_updated.md | 802 ++++++++++++++++++ .../lmeval_custom_tasks/sklegal.yaml | 0 .../lmeval_custom_tasks/skquad.yaml | 0 .../lmeval_custom_tasks/task.py | 0 lmeval/lmeval_comparison_all.csv | 14 + .../plots/lmeval_main_metrics_chart.pdf | Bin .../plots/lmeval_main_metrics_chart.png | Bin .../results/lmeval_results_full_table.csv | 0 .../results_2026-08-17T21-53-33.966513.json | 406 +++++++++ .../results_2026-08-18T14-34-57.786558.json | 208 +++++ .../results_2026-09-30T21-23-33.435952.json | 432 ++++++++++ .../results_2026-10-01T04-23-23.791749.json | 432 ++++++++++ .../results_2026-10-01T10-25-51.531765.json | 432 ++++++++++ .../results_2026-10-01T17-22-48.146390.json | 432 ++++++++++ .../results_2026-10-02T00-19-55.725179.json | 432 ++++++++++ .../results_2026-10-02T06-59-24.237777.json | 432 ++++++++++ .../results_2026-10-02T14-27-49.730611.json | 432 ++++++++++ .../results_2026-10-02T19-20-02.720498.json | 432 ++++++++++ .../results_2026-10-03T01-58-48.046423.json | 432 ++++++++++ .../results_2026-10-03T07-58-30.464881.json | 432 ++++++++++ .../results_2026-10-01T21-46-30.546755.json | 432 ++++++++++ .../results_2026-10-01T01-37-58.346736.json | 432 ++++++++++ 23 files changed, 6614 insertions(+) delete mode 100644 README.md create mode 100644 README_lmeval_updated.md rename lmeval/{evaluation => costom}/lmeval_custom_tasks/sklegal.yaml (100%) rename lmeval/{evaluation => costom}/lmeval_custom_tasks/skquad.yaml (100%) rename lmeval/{evaluation => costom}/lmeval_custom_tasks/task.py (100%) create mode 100644 lmeval/lmeval_comparison_all.csv rename lmeval/{ => mistral_alpaca(titan)}/plots/lmeval_main_metrics_chart.pdf (100%) rename lmeval/{ => mistral_alpaca(titan)}/plots/lmeval_main_metrics_chart.png (100%) rename lmeval/{ => mistral_alpaca(titan)}/results/lmeval_results_full_table.csv (100%) create mode 100644 lmeval/mistral_alpaca(titan)/results/results_2026-08-17T21-53-33.966513.json create mode 100644 lmeval/mistral_alpaca(titan)/results/results_2026-08-18T14-34-57.786558.json create mode 100644 lmeval/mistral_aya(hruska)/01_baseline_r16_a16_lr5e5_adamw/results_2026-09-30T21-23-33.435952.json create mode 100644 lmeval/mistral_aya(hruska)/02_optimizer_adamw8bit_r16_a16_lr5e5/results_2026-10-01T04-23-23.791749.json create mode 100644 lmeval/mistral_aya(hruska)/03_rank8_a16_lr5e5/results_2026-10-01T10-25-51.531765.json create mode 100644 lmeval/mistral_aya(hruska)/04_rank32_a32_lr5e5/results_2026-10-01T17-22-48.146390.json create mode 100644 lmeval/mistral_aya(hruska)/05_r16_alpha32_lr5e5/results_2026-10-02T00-19-55.725179.json create mode 100644 lmeval/mistral_aya(hruska)/06_lr1e4_r16_a16/results_2026-10-02T06-59-24.237777.json create mode 100644 lmeval/mistral_aya(hruska)/07_lr2e5_r16_a16/results_2026-10-02T14-27-49.730611.json create mode 100644 lmeval/mistral_aya(hruska)/08_attention_only_r16_a16_lr5e5/results_2026-10-02T19-20-02.720498.json create mode 100644 lmeval/mistral_aya(hruska)/09_batch2_accum4_r16_a16_lr5e5/results_2026-10-03T01-58-48.046423.json create mode 100644 lmeval/mistral_aya(hruska)/10_r16_a32_lr1e4/results_2026-10-03T07-58-30.464881.json create mode 100644 lmeval/mistral_aya(titan)/titan_aya_25k/results_2026-10-01T21-46-30.546755.json create mode 100644 lmeval/mistral_aya(titan)/titan_aya_full/results_2026-10-01T01-37-58.346736.json diff --git a/README.md b/README.md deleted file mode 100644 index 77c4bb6ce15d48c4a2a7035f2df085f7bcbe15b8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 28578 zcmeI5%Wqvra>fs{BrxzIS!QuzW3ri{MACXtMr=TqWXoe&)>yJ82?FpXTB7)n%-4(+ z$Ny%4?6}lIt7K)sNxolyx7gk1_I*&2jW`Ghk@ud{r@Fe{Rb74b-~aVqH|Rd=_PfLG zs5|J6yG{Mw?w;w&p!-Ij_PecaLD!D;iEoc}{jl5Bh!fr6xt;E1_e9?Z-4_~RIO`fQ z=&p7j>-V1SJ<`}+!I;<2d-@r4_jL8R`(bWduz#vXG3L6U0`Wz6s!zM!x*+WL#tpg+ zjbWbW6|4`^EDO56c82M?pezW&uEt&zMkSsLX$=d)c0*4v`$Nsd8n@17W&U+Nd!)4< zB)pjQKs1Q8Z*_msl`UP{7JMiJ%?905eIDrPXNfY$8sl~B3*Ml+tuc@F6O{c#S2y)U zxrRUJ+K$F;bW7d)-Ey}$aRsc#-0_;>#PbAoT@?5>-8;~CuXI7z7Sfu{mD{>=AZ%IJ zLArjcv99lgxoN+xPY1%6S+>P3q+(qwe5vbRA+VllO;2>?P$QukzkTh(+1&5x{sX}{ zO3%XuIJn&Xn||LFM?ULr>fiM=>NAbJuB(IYD}DNhv$2Y`08X&iKaJs5QNlLWw}9gVxLZy_D5udSQN zK74_HM~M!1^$mU^&*mu_g}#ip8lkPw+q&~in$pGW4LD7+m)iG@<~tA_xl?K%R4%zYNObxst;rNw(_CwM z3R!`xA@iSAG~GN~%Y}}gkdpTV|6y7aG62NU`(KJ~&;yNsXRaLMxdRrS1IhXijYqDq zmhkkYFo`umQO19vEA}Qqh4y%)gSqiOoBD?>K99UI9!vb9;N#bci&)nqaooInAWL&c zW7nivpU4_L5Z=fbQgd*|&3oOgba$Y!&ysEO(|ILI`@Npt(eua;)IyG}ncJe=spOgOSc%#Tztp@?r}XQrzt)~vd~KE{EG;^D zqE+BKp2#xWvS6K{C_a_7+A>V{yvWM;f^@HX9-a)+_1juEF$KJ2o!_VNwx{1Guec!2 z`|3d4LzeJjhr{QW&YnSf*QG;Ha7!NcSikp^>AY)W`Y zyeMk~_i~O;6BQmwvK(8$%}xCrCvHHy9Jlf8N-@JN`HXM$oMSt*5#1R#Yx(3IjfMU> z_JbGL3}d*gPYcy*z7(avgcysJ1ih~(+1u;>lyFXF&R?oe5B=3P8xG# zBOTL1FJ`lE+U#B>*|qI_CHT%)BIxsR5{a9p_24NqWpSbu-#k-%%Zo$Qm zB@?&wozFoRY!jM=&e-mwtIu_ncmUn}Zs_gN6#HlBMkH)GEWOdQH0xv=nsHhfTDE~S z<$`jWtNLE*YdO-I=aEYqiS^97RgSzf#0!|q8d+;?(7?D~Nn=)0te&CW8p=#b8&(g* z75!eREb={3s!uUPUrx|334^c$5n0bJ)4S5vg)?p;#bGxVB;mHnNa119n9`e}C&+5V z+tVX~?Df(6cz!m|-;O2E8w<>LztcK?o=mrIS>6+E?n(Be`haEJNUO#N?(l$|K=B%6r) z#44xqAz%jtp7<%@j>RS##99)m1f(q6Yg!vM7;Mv_Z11|Fk{TmC?Vf(9{Se9ccjWhn z>AIs#TZ}xn?c!6-jDI~zYXA$Z{D$Cc5lgDlvo*BM+`|fEnalXT#CA{EfejYC#s?ZY#|O^(_CU0c&#YCA5N-m7FeTCBnq(K2f}cpZr!hdUmkcl>P_*#(Acoz_xJm zyxSvud6g_N(aUbqYFl0M4WMr8FJA78?nY&A$kirexBLrZS6X8H8J=%Ta)sXwI`z4Q zwjn2{qK{*`Hdm6vY^+4*uwU*U@wX~CAQ;jBX}@6qNYBk%cA3Q zl^agCD%s{;72TX2;sMA$GgX&0^R?HuSCb$E_fi%0we$}@vijgd3&Rz`DRb)cz-jU3 zQj*&DE8ev1(KSiViZC;^h_9huWQUE^=yC`J-aB^=@m1Z zA0Rg+E+ATps-z*iE{V5(ES!jrh>rcVEA6BBIV)Mvs5L!-WIJ}syPM~&y{`*PGJf&{ zVhFGx7651B>St*$lPCaB4`sPV47#r<7aSPPeWJepXuK3-pQh(e(@gPwPtSV1VVHAN zk{mC3$L-aScmF9ku9+BG-W@p-gZto|pDah#?o35a_s8( zB#fEQ*>1)ydKNN8>N!ja@C}*dSu7)3GtfGbtu4hNe5-YbD?#6u&f2a$5DnO$!Ed+X z7}p1XNID&Sp-#%qguU#v4i$TT_<){UgYc-1OW0$>vbqM*UjaCNt|#|ZliE&lPW6O6 z4Q$AvxCs3Z^ckGOGlVCePg>#~X=+$y=TlhBupdERbbfVM0o&ClECIIqLb5Z|n? zJkWmjx1xf|+-1Ih-aYhk-t+CDmrrJzqLbw=HFlQWudugym3K7qeEaTibN}2OY#Eza z!Q*5Fu*C3_t|P`kV>0hN{&jl*vMf6?j9R26A*3gdvc9~neR}-Eth?^4y6y%tCHj)y z9}^TLj=|%&=IN+JVH)6Pz(S+14&=Gvug_n2>Gk{CJqe<+H=Y8)IX^koDrZpcgdBm5wqt zBp%JO@oVjUz@tv`rSz@78d3%ih-M|0-hCTcN@HM=jdrf307X%FnY;%FCbxYo@ zVXkUPUldh(r=8tU2L)$qsxYOc5I z1EbFmx(@}JI_70j_KGBI{21!IRL#eap~uelYy22?9jI?FO@PDRlXH5O~MGbax zmW6A5hy4Y2!1V9H33}S$Sp5t%p>qakdF6ML7VSt+iS$AbO1Z2h61nz@p~GI1c&b{? zFWS;ozl7Jawht^(i=QoP0Ny#gG8L5^x#SLbo^^~;R(x#l7*7oRcKX_X$qLCN-n7mpMecIG>PJ80 zYUNO6g=Dr9W;yHmCF>9FwJ^WWSAVE8=E&dCRrf)i=X^qb8c&ex7j>;TQ)pW4;MS#6$X4d{Dho9|Gqmw105Uz!TUTgR$NEzIf3fTTfuff(11onY@QmbK z^g!SZvDtj@$2^uv$Ng)Sb#>xX(EH--+mA)YYU#QKgAd^n{e7(1^0~665HpD(7_Ol{3~D zN54)~TzsvK9a%%!3k9~9g>4@{#+6lBa)%avj4I)c+8a}a>jBSE$y51R&fBi5QbI)K zT$mU(PKah1h}fG*k0{q5$qo*Yk--}fteWh}%D?hIxZfe3=g zyzb_RNS}KV+c~xz?NFtvLS%_FgCo1EVDItNf*e_$N^96NqV@v@K1E4n*@_X%lh5|f zwyfnW^zedrM4hcD34F#!i}ft#1(6iJLPz?|il8hJ(V$uBMEenWw05 zL5t{ADp%lq`b4zjOcb#q3g)F7WP?JD{IXnoi6UDxX@ z(`1#CB6jl!+EX#4+KgWw)m$uaP!c(h{HRYwVpFf5cNEx0`aEBIPo63yu$+ShKhYXW`*f#k7IiHKDC6||*PBO= z>ojOtld_9oTGl)I4wD>dVu7(M{Wk#-4`$yGap^2M__!=5;+BAwv(LAgyK+LpQEbff zCi(T-%wHW>X3EF{v(U#+&D+@vfPSP0o3c2kHUS_I!iRQJc1R z32Yyp={OO?th?FwH7~xx6v)|$D!g;p_x%(l@E#-E)l8Ed3I4iH#M7;fX0XHHJ~upW z+qxkWE2(B3ee}*I%HA{f?1|Fa(aM{=1)!|5=eG%kr*y_(u8i}yVVQ3!BI=(gFvsgk z=?hUMWdlhb$g`bw=fBRq*F={k?N_pHq7C*sv1zN4Vd^`py6(Tbxy^?63sR=EB}G;%#js^5wcIsu-`nmNbi?(I0Juvrlv z_{>Sz2gc_~cAv2{1+IB_lx(wi8|ZaFns_4*efafEUY(^qX3ZS7m2hkYH~MfL8~d#R zJ1I&dR`a`VvesSGtT`vNG(^@nO4Alb`Dr5W=ykG;glC+dQ?esdN)QoKNslZ!=628n zOV!_ZZpr5F1buc%$nS*6ZyWUfaa|Re7Qv3UlT=jQA>iG0zwLvlA^Y)Ntu0^ws<-=U zn^fD=x~|!;a3X_m!^#rZa8~u2;MwYPg^J=8{q~is@?UfseV{Q^8h<_u8rXPGIpQv` zSr!Hv8)}&!=vrB~gd($BSI@+WKHJoRK6%|MT6vxcE76+iJeJpUQ6(_GF_%vX=eek` zexNb^Rh`fN#%#-H>+s6Zx|rV_<+u0qOODQFiGjJIpT686C)R%6ON!j^8#>o=Gn-xZ zT9Cwm>9ueI+FmMVK{|sj80l{OB3FO7NSwJ&PzffX{s3Gx;WZ*G;1L0d|Ha-`s6S)ZT_Vil=+HO6ru5>HYHw*6}A@$SHCa zzO2oh>Q2iKwsFf5M2@XK6a=i+d{HqdV>0ez3+N&nSLu!(K}S^c;zaa2MvUwcIpUoc ztXbcFmoap`jZ7rxFn&jdYkso!1!NKxsLrlQvmF5*EvpLQDtRkj0!oLPnT)DU#g$y2KPs^fCXXd?4qm&W;??BMac?~Xuk=Hoh3)mSe(7Y z8P|MuhHAKPJG$<`pv?G@v37PqTy`y8c#>Aac}>nJ+ZVRxGIfBV}`Kgeh6L?0_y)M_{(8M}618c)61djCrE@)OYBESmzaQHXD+ zy3~d}J=1AFH%s2qesZ=fr2XvVoTB~A^?B2=CUiLB+j zrOCWQqaQ=tqhcqp7LVl(`JWITpA#;7jd7MaZ02~#vAS6>=6nG8|cdu>A{$MS$ z{rOz?d`7NqFGu&}+qUWNo7VOt&55<~>FT`BpcbozTKzjYOUV=+9W%<0JUUkBN=}Y> zzZ&{AauK@W-GAKg@W)HUzHLhu!sq-!(BcFUm3QyIw4T5pd3S>@CH~h0_Qqc%-{ALF zICjqO5Mms=^LWqFQw3iCs5<)>$~s2(_@g7tHB0=7{V7=I4hQcw)m`j<-wu?1DLFQm znBSO_IdPA30LI@P{eCMXy5aX9kwHYCg1gb(S#U?#t_$bus_^>%1aU_Wv>{F+-XNCJ zK889Vi1W%G4Ep8mNl*2+-sN#r$J^_nn0Ly687J`^^^zeF#h?%PIp24ru__w#{skn` z@4?^Hd_+n3b}7%ocVj?)#mM5Mu) zz0yX6-ny!RMC6l}At7!0l{6s7@&ErY%?Fg63e80Ba+P2!{T~l8?tdF~;m0HyMgOob zmX;IIf4xV4BGTHIphKJ)$7^gU7jt|&8XZofZJV#xg#+0UwU-Tf2Xx(a1$Q)2wX*#s zk0Yk{`@FE0Xg>G0y4!lHwu)gf>{IHv{r|a^WBS>-W^G^YKI;B1={`FIaEk2q{4!_0 zAZ^nz@Yv8fyth3#Z+$0Q!wx&D39soab=G{*{^o&n!<`k;^U3~s|NZ&cW#@}@>_&#^ zenh$`@ehsxqLMBT|?R(w7F}>s4v6C^)vSUmgzYt4+3z7mx_DNJi z`xW_&mvQO8r7E8R@&CAxt3*UC$edS97YyG=d-GXq^d}L+dB4W}{a(pv#RS=TD%#-z leTIboCG7OEs7TF(s0@DsC$gOTI=UEdAaI9lOIMlue*h!w9)17- diff --git a/README_lmeval_updated.md b/README_lmeval_updated.md new file mode 100644 index 0000000..30e44df --- /dev/null +++ b/README_lmeval_updated.md @@ -0,0 +1,802 @@ +# Instruction fine-tuning slovenského Mistral 7B modelu + +## Použitý model + +Ako základný model bol použitý: + +- `slovak-nlp/mistral-sk-7b` + +Výsledný adaptér z prvého experimentu je dostupný na Hugging Face: + +- `Jakub1320/mistral-sk-7b-slovak-alpaca-qlora` + +--- + +## Hardvér + +Experimenty boli realizované na dvoch serveroch. + +### Titan + +- 2× NVIDIA TITAN RTX +- 24 GB VRAM na GPU +- výpočty v `fp16` + +### Hruška + +- 2× NVIDIA RTX A4500 +- približne 20 GB VRAM na GPU +- výpočty v `bf16` + +--- + +# Prvý experiment – Slovak Alpaca + +## Použitý dataset + +Na supervised fine-tuning bol v prvom experimente použitý slovenský inštrukčný dataset: + +- `saillab/alpaca-slovak-cleaned` + +Dataset obsahuje polia: + +- `instruction` +- `input` +- `output` + +Dáta boli formátované v Alpaca štýle: + +```text +### Inštrukcia: +... + +### Vstup: +... + +### Odpoveď: +... +``` + +## Metóda dotrénovania + +Model bol dotrénovaný pomocou supervised fine-tuningu cez: + +- Unsloth +- TRL `SFTTrainer` +- PEFT LoRA +- QLoRA štýl trénovania + +Výsledkom tréningu je PEFT LoRA adaptér, ktorý sa pripája k základnému modelu `slovak-nlp/mistral-sk-7b`. + +## Hlavné hyperparametre prvého experimentu + +| Parameter | Hodnota | +|---|---:| +| Epochs | 1 | +| Max sequence length | 1024 | +| Batch size | 1 | +| Gradient accumulation | 8 | +| Effective batch size | 8 | +| Learning rate | 2e-4 | +| Optimizer | AdamW 8-bit | +| Scheduler | cosine | +| Warmup steps | 150 | +| LoRA rank | 16 | +| LoRA alpha | 32 | +| LoRA dropout | 0.05 | +| Quantization | 4-bit | +| Compute type | fp16 | + +Použité LoRA cieľové moduly: + +```text +q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj +``` + +## Vyhodnotenie pomocou vlastného evaluačného skriptu + +Model bol vyhodnotený na testovacích príkladoch z datasetu `saillab/alpaca-slovak-cleaned` pomocou metrík ROUGE, BLEU, eval loss a perplexity. + +| Metrika | Výsledok | +|---|---:| +| ROUGE-1 | 0.3710 | +| ROUGE-2 | 0.1702 | +| ROUGE-L | 0.2732 | +| BLEU | 6.2850 | +| Eval loss | 0.8081 | +| Perplexity | 2.2435 | + +## Vyhodnotenie pomocou lm-evaluation-harness + +Hodnotený bol PEFT LoRA adaptér pripojený k základnému modelu: + +```text +slovak-nlp/mistral-sk-7b ++ +Jakub1320/mistral-sk-7b-slovak-alpaca-qlora +``` + +### Použité benchmarky + +| Benchmark | Popis | +|---|---| +| `arc_sk` | slovenské vedecké/školské otázky s výberom odpovede | +| `hellaswag_sk` | commonsense reasoning a výber najpravdepodobnejšieho pokračovania | +| `m_mmlu_sk` | slovenský všeobecný vedomostný benchmark | +| `truthfulqa_sk_mc1` | hodnotenie pravdivosti odpovedí, multiple choice MC1 | +| `truthfulqa_sk_mc2` | hodnotenie pravdivosti odpovedí, multiple choice MC2 | +| `sklegal` | slovenské právne otázky s výberom odpovede | +| `skquad` | slovenský question-answering benchmark | + +Úlohy `sklegal` a `skquad` boli doplnené ako custom tasky. + +## Výsledky lm-evaluation-harness + +Poznámka: výsledky `arc_sk`, `hellaswag_sk`, `m_mmlu_sk` a TruthfulQA pochádzajú z prvého behu. `sklegal` a `skquad` boli následne dopočítané v druhom samostatnom behu nad tým istým Alpaca QLoRA adaptérom, pretože v pôvodnom behu tieto dve custom úlohy nefungovali. + +| Task | Počet vzoriek | Metrika | Výsledok | +|---|---:|---|---:| +| `arc_sk` | 1169 | acc | 0.3259 | +| `arc_sk` | 1169 | acc_norm | 0.3473 | +| `hellaswag_sk` | 9485 | acc | 0.4134 | +| `hellaswag_sk` | 9485 | acc_norm | 0.5106 | +| `m_mmlu_sk` | 13062 | acc | 0.2928 | +| `truthfulqa_sk_mc1` | 778 | acc | 0.2391 | +| `truthfulqa_sk_mc2` | 778 | acc | 0.3942 | +| `sklegal` | 1334 | acc | 0.2841 | +| `sklegal` | 1334 | acc_norm | 0.4693 | +| `skquad` | 9583 | exact | 1.9722 | +| `skquad` | 9583 | f1 | 27.7994 | +| `skquad` | 9583 | HasAns_exact | 2.4106 | +| `skquad` | 9583 | HasAns_f1 | 34.1456 | +| `skquad` | 9583 | NoAns_exact | 0.0561 | +| `skquad` | 9583 | NoAns_f1 | 0.0561 | +| `skquad` | 9583 | best_exact | 18.6163 | +| `skquad` | 9583 | best_f1 | 28.3012 | + +--- + +# Experiment so strojovým prekladom datasetu + +Cieľom experimentu bolo vyskúšať postup, pri ktorom sa anglické polia datasetu: + +- `instruction` +- `input` +- `output` + +automaticky preložia do slovenčiny a uložia do novej dátovej množiny so zachovaním pôvodnej štruktúry. + +Výstupný formát obsahoval pôvodné anglické texty aj slovenské preklady: + +```json +{ + "instruction_en": "...", + "input_en": "...", + "output_en": "...", + "instruction_sk": "...", + "input_sk": "...", + "output_sk": "..." +} +``` + +Vznikla preložená vzorka datasetu s 1000 príkladmi. + +Umiestnenie: + +```text +strojovy_preklad/translated_datasets/alpaca_translated_en_sk_1000.jsonl +``` + +--- + +# Experiment s datasetom Aya Collection + +Pre ďalšie experimenty bol použitý dataset: + +- `CohereLabs/aya_collection_language_split` +- subset: `slovak` + +Používané polia boli mapované nasledovne: + +| Pôvodné pole | Použitie pri tréningu | +|---|---| +| `inputs` | prompt / inštrukcia | +| `targets` | odpoveď | + +--- + + +# Prvý dlhý Aya tréning na serveri Titan – manuálne ukončený pokus + +Pred úspešným 225 000-krokovým tréningom bol na serveri Titan spustený prvý dlhý tréning nad slovenskou časťou datasetu Aya Collection. + +Tento experiment bežal na jednej NVIDIA TITAN RTX a používal streaming datasetu. +Konfiguráčny súbor je +```text +train_mistral_sk_aya_slovak_full_lora.yaml +``` + +## Konfigurácia prvého dlhého tréningu + +| Parameter | Hodnota | +|---|---:| +| Base model | `slovak-nlp/mistral-sk-7b` | +| Dataset | `aya_slovak_full` | +| Template | `alpaca` | +| Fine-tuning | LoRA | +| GPU | 1× NVIDIA TITAN RTX | +| Max sequence length | 1024 | +| Streaming | áno | +| Streaming buffer | 16 384 | +| Batch size per device | 1 | +| Gradient accumulation | 4 | +| Effective batch size | 4 | +| Max steps | 1 037 000 | +| Learning rate | 2e-4 | +| Scheduler | cosine | +| Warmup steps | 500 | +| Max grad norm | 1.0 | +| Optimizer | AdamW 8-bit | +| LoRA rank | 16 | +| LoRA alpha | 32 | +| LoRA dropout | 0.05 | +| Compute type | fp16 | +| Unsloth | áno | +| Unsloth gradient checkpointing | áno | +| Validation size | 1000 | +| Eval interval | každých 50 000 krokov | +| Save interval | každých 10 000 krokov | +| Save total limit | 2 | + +Použité LoRA cieľové moduly: + +```text +q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj +``` + + + +Konfigurácia počítala približne s jedným prechodom cez celý slovenský Aya subset: + +```text +4 148 187 príkladov / effective batch 4 ≈ 1 037 000 krokov +``` + +## Dôvod manuálneho ukončenia + +Tréning technicky nespadol. Po približne 94 000 krokoch sa však začala výrazne zhoršovať stabilita optimalizácie. + +V logu sa objavili extrémne hodnoty `grad_norm`, napríklad: + +```text +grad_norm = 125.6 +grad_norm = 105.9 +grad_norm = 24000 +grad_norm = 772.1 +grad_norm = 11410 +grad_norm = 2714 +grad_norm = nan +``` + +Súčasne začal výrazne kolísať aj training loss, napríklad: + +```text +loss = 3.232 +loss = 7.081 +loss = 5.206 +loss = 14.82 +``` + +Neskôr sa v logu objavila aj kombinácia: + +```text +loss = 0.0 +grad_norm = nan +``` + +Keďže tréning vykazoval známky numerickej nestability, bol manuálne ukončený približne pri kroku `104 238` z plánovaných `1 037 000`, teda približne po 10 % tréningu. + +V čase ukončenia bol runtime približne: + +```text +44 hodín 48 minút +``` + +Tento experiment ukázal, že pôvodné nastavenie nebolo vhodné na pokračovanie v takto dlhom tréningu bez ďalších úprav. + +--- + +# Dlhý Aya tréning na serveri Titan + +Na serveri Titan bol spustený dlhý LoRA fine-tuning nad slovenskou časťou Aya Collection. + +Na rozdiel od prvého QLoRA experimentu nebola použitá 4-bit kvantizácia. + +## Hlavná konfigurácia + +| Parameter | Hodnota | +|---|---:| +| Base model | `slovak-nlp/mistral-sk-7b` | +| Dataset | Aya Collection – Slovak | +| Fine-tuning | LoRA | +| Unsloth | áno | +| GPU | 2× NVIDIA TITAN RTX | +| Batch size per device | 1 | +| Gradient accumulation | 8 | +| Effective batch size | 16 | +| Max sequence length | 1024 | +| Max steps | 225 000 | +| Learning rate | 5e-5 | +| Scheduler | cosine | +| Warmup steps | 10 000 | +| LoRA rank | 16 | +| LoRA alpha | 16 | +| LoRA dropout | 0.0 | +| Compute type | fp16 | +| Streaming | áno | + +Použité LoRA cieľové moduly: + +```text +q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj +``` + +Tréning úspešne dobehol do kroku `225000`. + +Pri effective batch size 16 bolo do tohto bodu spracovaných približne: + +```text +225 000 × 16 = 3 600 000 príkladov +``` + +To predstavuje približne 86,8 % slovenského Aya datasetu s približne 4,148 milióna príkladmi. Nešlo teda o úplný jeden epoch. + +## Eval loss počas Titan tréningu + +| Krok | Eval loss | +|---:|---:| +| 25 000 | 1.0599 | +| 50 000 | 1.174 | +| 75 000 | 1.284 | +| 100 000 | 1.429 | +| 125 000 | 1.342 | +| 150 000 | 1.323 | +| 175 000 | 1.321 | +| 200 000 | 1.213 | +| 225 000 | 1.221 | + +Najnižší validačný loss bol zaznamenaný pri kroku `25000`. Tento checkpoint bol samostatne zachovaný pre ďalšie benchmarkové porovnanie s finálnym modelom. + +--- + +# Hruška – 10 porovnávacích experimentov + +Na serveri Hruška bola vytvorená séria menších experimentov s cieľom porovnať vplyv vybraných hyperparametrov LoRA fine-tuningu. + +Cieľom bolo porovnať vplyv: + +- learning rate, +- LoRA rank, +- LoRA alpha, +- optimalizátora, +- LoRA target modulov, +- batch size a gradient accumulation. + +## Fixný dataset + +Pre férové porovnanie všetkých experimentov bol vytvorený fixný dataset: + +```text +aya_slovak_100k_fixed.jsonl +``` + +Dataset obsahuje presne `100 000` slovenských príkladov z Aya Collection. + +Pri jeho vytvorení bol použitý fixný seed. Samotné experimenty už nepoužívali streaming, takže všetky konfigurácie pracovali s rovnakým poolom dát. + +Dáta boli uložené vo formáte: + +```json +{ + "instruction": "...", + "input": "", + "output": "..." +} +``` + +Pri tréningu bolo `1000` príkladov použitých na validáciu. + +## Spoločné nastavenia experimentov + +| Parameter | Hodnota | +|---|---:| +| Base model | `slovak-nlp/mistral-sk-7b` | +| Dataset | `aya_slovak_100k_fixed` | +| GPU | 2× NVIDIA RTX A4500 | +| Fine-tuning | LoRA | +| Unsloth | áno | +| Streaming | nie | +| Max sequence length | 1024 | +| Max steps | 6250 | +| Effective batch size | 16 | +| Scheduler | cosine | +| Warmup steps | 500 | +| Max grad norm | 0.5 | +| LoRA dropout | 0.0 | +| Compute type | bf16 | +| Eval interval | každých 1250 krokov | +| Validation size | 1000 | +| Seed | 42 | + +Štandardné LoRA cieľové moduly: + +```text +q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj +``` + +Výnimkou je experiment 08, ktorý používa iba: + +```text +q_proj, k_proj, v_proj, o_proj +``` + +## Porovnávané konfigurácie + +| Experiment | Hlavná zmena | +|---|---| +| `01_baseline_r16_a16_lr5e5_adamw` | baseline: rank 16, alpha 16, LR 5e-5, AdamW | +| `02_optimizer_adamw8bit_r16_a16_lr5e5` | AdamW 8-bit | +| `03_rank8_a16_lr5e5` | LoRA rank 8 | +| `04_rank32_a32_lr5e5` | LoRA rank 32, alpha 32 | +| `05_r16_alpha32_lr5e5` | alpha 32 pri rank 16 | +| `06_lr1e4_r16_a16` | learning rate 1e-4 | +| `07_lr2e5_r16_a16` | learning rate 2e-5 | +| `08_attention_only_r16_a16_lr5e5` | LoRA iba na attention projekciách | +| `09_batch2_accum4_r16_a16_lr5e5` | batch size 2, gradient accumulation 4 | +| `10_r16_a32_lr1e4` | kombinácia alpha 32 + learning rate 1e-4 | + +## Výsledky Hruška experimentov + +| Experiment | Train loss | Eval loss | Runtime | +|---|---:|---:|---:| +| `01_baseline_r16_a16_lr5e5_adamw` | 0.9290 | 0.7567 | 7:18:05 | +| `02_optimizer_adamw8bit_r16_a16_lr5e5` | 0.9248 | 0.7521 | 7:20:23 | +| `03_rank8_a16_lr5e5` | 0.9308 | 0.7586 | 7:16:23 | +| `04_rank32_a32_lr5e5` | 0.9003 | 0.7292 | 7:22:33 | +| `05_r16_alpha32_lr5e5` | 0.9026 | 0.7323 | 7:17:44 | +| `06_lr1e4_r16_a16` | 0.8890 | 0.7184 | 7:17:30 | +| `07_lr2e5_r16_a16` | 0.9674 | 0.7912 | 7:18:25 | +| `08_attention_only_r16_a16_lr5e5` | 0.9808 | 0.8094 | 6:51:25 | +| `09_batch2_accum4_r16_a16_lr5e5` | 0.9290 | 0.7563 | 8:24:57 | +| `10_r16_a32_lr1e4` | 0.8612 | 0.6918 | približne 7:15 h | + +Najnižší finálny `eval_loss` dosiahol experiment: + +```text +10_r16_a32_lr1e4 +``` + +s hodnotou: + +```text +eval_loss = 0.6918 +``` + +V tomto experimente bola použitá kombinácia: + +```text +LoRA rank = 16 +LoRA alpha = 32 +learning rate = 1e-4 +optimizer = AdamW +effective batch size = 16 +``` + +## Pozorované rozdiely + +Z doterajších výsledkov: + +- zvýšenie learning rate z `5e-5` na `1e-4` znížilo eval loss z `0.7567` na `0.7184`, +- zvýšenie LoRA alpha z `16` na `32` pri rank 16 znížilo eval loss na `0.7323`, +- kombinácia `alpha=32` a `learning_rate=1e-4` dosiahla najnižší eval loss `0.6918`, +- rank 32 priniesol eval loss `0.7292`, +- rank 8 dosiahol výsledok veľmi blízky baseline, +- attention-only LoRA bola rýchlejšia, ale mala vyšší eval loss, +- zmena z batch size 1 / accumulation 8 na batch size 2 / accumulation 4 nepriniesla zlepšenie validačného lossu ani runtime. + +--- + +# Súbory Hruška experimentov + +Výsledky experimentov sú uložené v priečinku: + +```text +hruska_10_experiments_results/ +``` + +Súčasťou sú: + +```text +hruska_10_experiments_results/ +├── aya_slovak_100k_fixed/ +├── hruska_10_experiments_results/ +│ ├── 01_baseline_r16_a16_lr5e5_adamw/ +│ ├── 02_optimizer_adamw8bit_r16_a16_lr5e5/ +│ ├── 03_rank8_a16_lr5e5/ +│ ├── 04_rank32_a32_lr5e5/ +│ ├── 05_r16_alpha32_lr5e5/ +│ ├── 06_lr1e4_r16_a16/ +│ ├── 07_lr2e5_r16_a16/ +│ ├── 08_attention_only_r16_a16_lr5e5/ +│ ├── 09_batch2_accum4_r16_a16_lr5e5/ +│ └── 10_r16_a32_lr1e4/ +├── hruska_experiment_eval_loss_ranking.png +├── hruska_experiment_train_loss_ranking.png +└── hruska_experiment_summary.csv +``` + +Každý experiment obsahuje: + +```text +.yaml +training.log +training_loss.png +training_eval_loss.png +``` +--- + +## Reprodukcia tréningových experimentov + +Jednotlivé experimenty je možné zreplikovať pomocou uložených YAML konfiguračných súborov. + +Pred spustením tréningu je potrebné aktivovať prostredie s LLaMA-Factory a prejsť do jeho adresára: + +```bash +conda activate llama_factory_unsloth +cd /home/schwarc/LLaMA-Factory +``` + +Pre 2-GPU experimenty bol tréning spúšťaný pomocou: + +```bash +CUDA_VISIBLE_DEVICES=0,1 FORCE_TORCHRUN=1 \ +llamafactory-cli train /cesta/ku/configu.yaml +``` + +Napríklad pre experiment: + +```text +10_r16_a32_lr1e4 +``` + +možno použiť: + +```bash +CUDA_VISIBLE_DEVICES=0,1 FORCE_TORCHRUN=1 \ +llamafactory-cli train \ +/home/schwarc/diplomovka/configs/hruska_compare9_100k_fixed/10_r16_a32_lr1e4.yaml +``` + +Ak má byť terminálový výstup zároveň uložený do logu: + +```bash +CUDA_VISIBLE_DEVICES=0,1 FORCE_TORCHRUN=1 \ +llamafactory-cli train \ +/home/schwarc/diplomovka/configs/hruska_compare9_100k_fixed/10_r16_a32_lr1e4.yaml \ +2>&1 | tee /home/schwarc/diplomovka/logs/10_r16_a32_lr1e4.log +``` + + + +# Výsledky benchmarkovania pomocou lm-evaluation-harness + +Po dokončení tréningov boli vyhodnotené: + +- všetkých 10 Hruška experimentov, +- Titan checkpoint po 25 000 krokoch, +- finálny Titan model po 225 000 krokoch, +- starší QLoRA model trénovaný na `saillab/alpaca-slovak-cleaned`. +- Kompletné výsledky a výstupné súbory z evaluácie pomocou lm-evaluation-harness sú dostupné v priečinku lmeval/ + +Použité benchmarky: + +```text +arc_sk +hellaswag_sk +m_mmlu_sk +truthfulqa_sk_mc1 +truthfulqa_sk_mc2 +sklegal +skquad +``` + +Počet hodnotených vzoriek bol: + +| Benchmark | Počet vzoriek | +|---|---:| +| `arc_sk` | 1169 | +| `hellaswag_sk` | 9485 | +| `m_mmlu_sk` | 13062 | +| `truthfulqa_sk_mc1` | 778 | +| `truthfulqa_sk_mc2` | 778 | +| `sklegal` | 1334 | +| `skquad` | 9583 | + +Pri úlohách, kde `lm-evaluation-harness` poskytuje `acc` aj `acc_norm`, je v súhrnných tabuľkách použitá najmä hodnota `acc_norm`. Pri `skquad` je ako hlavná súhrnná metrika použitá hodnota `F1`. + +Hruška benchmarky boli počítané v `bfloat16`, Titan a starší Alpaca model v `float16`. Batch size pri evaluácii bol 1. + +## Poznámka k staršiemu Alpaca benchmarku + +Starší model `Jakub1320/mistral-sk-7b-slovak-alpaca-qlora` bol vyhodnotený v dvoch behoch. V prvom boli vyhodnotené: + +```text +arc_sk +hellaswag_sk +m_mmlu_sk +truthfulqa_sk_mc1 +truthfulqa_sk_mc2 +``` + +Úlohy `sklegal` a `skquad` vtedy nefungovali, preto boli neskôr spustené samostatne. Výsledky uvedené v README sú zlúčením týchto dvoch behov nad tým istým adaptérom. + +Pri porovnávaní starého Alpaca modelu s novšími behmi treba brať do úvahy, že bol použitý iný softvérový stav. Najdôležitejšie je, že starý výsledok `truthfulqa_sk_mc2` má v uloženom výsledku verziu úlohy 2.0, zatiaľ čo novšie Hruška a Titan behy používajú verziu 1.0. Preto výsledok MC2 starého Alpaca modelu nie je vhodné považovať za úplne priamo porovnateľný s novými behmi. Ostatné hlavné verzie benchmarkov sa zhodujú. + +--- + +## Výsledky 10 Hruška experimentov + +| Experiment | Eval loss | ARC acc_norm | HellaSwag acc_norm | M-MMLU acc | TQA MC1 | TQA MC2 | SKLegal acc_norm | SKQuad F1 | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| `01_baseline_r16_a16_lr5e5_adamw` | 0.7567 | 0.3405 | 0.5209 | 0.3550 | 0.1799 | 0.3439 | 0.4685 | 36.6797 | +| `02_optimizer_adamw8bit_r16_a16_lr5e5` | 0.7521 | 0.3379 | 0.5212 | 0.3544 | 0.1928 | 0.3537 | 0.4715 | 36.2178 | +| `03_rank8_a16_lr5e5` | 0.7586 | 0.3311 | 0.5167 | 0.3455 | 0.1838 | 0.3425 | 0.4715 | 36.3638 | +| `04_rank32_a32_lr5e5` | 0.7292 | 0.3353 | 0.5201 | 0.3450 | 0.1902 | 0.3367 | 0.4730 | 36.0228 | +| `05_r16_alpha32_lr5e5` | 0.7323 | 0.3336 | 0.5190 | 0.3571 | 0.1967 | 0.3555 | 0.4670 | 37.8735 | +| `06_lr1e4_r16_a16` | 0.7184 | 0.3311 | 0.5207 | 0.3550 | 0.1902 | 0.3473 | 0.4655 | 37.5796 | +| `07_lr2e5_r16_a16` | 0.7912 | 0.3276 | 0.5225 | 0.3379 | 0.1671 | 0.3246 | 0.4723 | 33.8381 | +| `08_attention_only_r16_a16_lr5e5` | 0.8094 | 0.3524 | 0.5218 | 0.3448 | 0.1877 | 0.3674 | 0.4768 | 43.3752 | +| `09_batch2_accum4_r16_a16_lr5e5` | 0.7563 | 0.3379 | 0.5209 | 0.3514 | 0.1864 | 0.3402 | 0.4693 | 35.3128 | +| `10_r16_a32_lr1e4` | 0.6918 | 0.3293 | 0.5157 | 0.3448 | 0.2057 | 0.3480 | 0.4618 | 37.7561 | + +### Najlepší výsledok Hruška podľa jednotlivých benchmarkov + +| Metrika | Najlepší Hruška experiment | Výsledok | +|---|---|---:| +| ARC acc_norm | `08_attention_only_r16_a16_lr5e5` | 0.3524 | +| HellaSwag acc_norm | `07_lr2e5_r16_a16` | 0.5225 | +| M-MMLU acc | `05_r16_alpha32_lr5e5` | 0.3571 | +| TruthfulQA MC1 | `10_r16_a32_lr1e4` | 0.2057 | +| TruthfulQA MC2 | `08_attention_only_r16_a16_lr5e5` | 0.3674 | +| SKLegal acc_norm | `08_attention_only_r16_a16_lr5e5` | 0.4768 | +| SKQuad F1 | `08_attention_only_r16_a16_lr5e5` | 43.3752 | + + +Najvýraznejší výsledok dosiahol experiment: + +```text +08_attention_only_r16_a16_lr5e5 +``` + +Ten mal síce zo všetkých 10 experimentov najvyšší, teda najhorší validačný `eval_loss = 0.8094`, ale súčasne dosiahol najlepší Hruška výsledok na: + +- `arc_sk` podľa `acc_norm`, +- `truthfulqa_sk_mc2`, +- `sklegal` podľa `acc_norm`, +- `skquad` podľa F1. + +Na `skquad` dosiahol F1 `43.3752`, zatiaľ čo baseline experiment 01 dosiahol `36.6797`. + +Naopak experiment: + +```text +10_r16_a32_lr1e4 +``` + +mal najlepší validačný `eval_loss = 0.6918`, ale nebol najlepší vo väčšine externých benchmarkov. Jeho najlepší výsledok v rámci Hruška experimentov bol `truthfulqa_sk_mc1 = 0.2057`. + +To ukazuje dôležitý rozdiel medzi validačným lossom a downstream benchmarkmi: nižší loss na validačnej časti Aya datasetu neznamená automaticky najlepší výkon na nezávislých úlohách. Validačný loss meria predovšetkým prispôsobenie dátovej distribúcii použitej pri tréningu, zatiaľ čo benchmarky merajú odlišné schopnosti modelu. + +Ďalšie pozorovania: + +- `07_lr2e5_r16_a16` dosiahol najlepší HellaSwag `acc_norm = 0.5225`, ale mal slabší eval loss `0.7912`. +- `05_r16_alpha32_lr5e5` dosiahol najlepší M-MMLU výsledok `0.3571`. +- AdamW 8-bit v experimente 02 dal benchmarkové výsledky veľmi podobné klasickému AdamW baseline. +- zmena batch size / gradient accumulation v experimente 09 nepriniesla jasné benchmarkové zlepšenie, +- rozdiely medzi viacerými Hruška modelmi na multiple-choice úlohách sú malé, preto ich netreba interpretovať ako zásadné bez ďalších behov alebo štatistického testovania, +- najvýraznejšie oddelenie experimentu 08 je na `skquad`, kde je rozdiel oproti ostatným modelom podstatne väčší. + +--- + +## Porovnanie Titan checkpointu 25k a finálneho modelu 225k + +Checkpoint 25 000 zodpovedá približne: + +```text +25 000 × effective batch size 16 = 400 000 spracovaných tréningových príkladov +``` + +Finálny krok 225 000 zodpovedá približne: + +```text +225 000 × 16 = 3 600 000 spracovaných tréningových príkladov +``` + +Slovenská časť Aya obsahuje približne 4,148 milióna príkladov, takže 225 000 krokov predstavuje približne 86,8 % jedného úplného prechodu datasetom, nie celý epoch. + +| Metrika | Titan 25k | Titan 225k | Zmena 225k − 25k | +|---|---:|---:|---:| +| ARC acc_norm | 0.3251 | 0.3447 | +1.97 p. b. | +| HellaSwag acc_norm | 0.4943 | 0.4793 | -1.50 p. b. | +| M-MMLU acc | 0.3342 | 0.2719 | -6.22 p. b. | +| TruthfulQA MC1 | 0.1761 | 0.2005 | +2.44 p. b. | +| TruthfulQA MC2 | 0.3251 | 0.3613 | +3.63 p. b. | +| SKLegal acc_norm | 0.4588 | 0.4393 | -1.95 p. b. | +| SKQuad F1 | 25.8678 | 35.0471 | +9.1793 bodu | + + +Výsledok Titan porovnania nie je jednoznačný. + +Finálny 225k model oproti 25k checkpointu zlepšil: + +- ARC `acc_norm` približne o 1,97 percentuálneho bodu, +- TruthfulQA MC1 približne o 2,44 p. b., +- TruthfulQA MC2 približne o 3,63 p. b., +- SKQuad F1 približne o 9,18 bodu. + +Naopak sa zhoršil: + +- HellaSwag `acc_norm` približne o 1,50 p. b., +- M-MMLU približne o 6,22 p. b., +- SKLegal `acc_norm` približne o 1,95 p. b. + +To je dôležité aj vzhľadom na validačný loss. Checkpoint 25k mal najlepší nameraný Titan eval loss približne `1.0599`, zatiaľ čo na konci bol eval loss približne `1.221`. Napriek tomu 225k checkpoint zlepšil niektoré downstream úlohy, najmä `skquad`. + +Preto sa nedá povedať, že checkpoint s najnižším validačným lossom je automaticky najlepší model pre všetky úlohy. + +--- + +## Súhrnné porovnanie vybraných modelov + +| Model / checkpoint | ARC acc_norm | HellaSwag acc_norm | M-MMLU acc | TQA MC1 | TQA MC2 | SKLegal acc_norm | SKQuad F1 | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| Alpaca QLoRA | 0.3473 | 0.5106 | 0.2928 | 0.2391 | 0.3942 * | 0.4693 | 27.7994 | +| Hruška 08 attention-only | 0.3524 | 0.5218 | 0.3448 | 0.1877 | 0.3674 | 0.4768 | 43.3752 | +| Hruška 10 best eval loss | 0.3293 | 0.5157 | 0.3448 | 0.2057 | 0.3480 | 0.4618 | 37.7561 | +| Titan 25k | 0.3251 | 0.4943 | 0.3342 | 0.1761 | 0.3251 | 0.4588 | 25.8678 | +| Titan 225k | 0.3447 | 0.4793 | 0.2719 | 0.2005 | 0.3613 | 0.4393 | 35.0471 | + +`*` Starý Alpaca výsledok `truthfulqa_sk_mc2` bol získaný s inou verziou tejto úlohy, preto ho treba porovnávať opatrne. + +Z porovnania možno vyvodiť: + +1. **Hruška experimenty sú veľmi konkurencieschopné napriek iba 100k dátam.** + Najmä experiment 08 dosiahol najlepší výsledok na `skquad` a silné výsledky na ARC a SKLegal. + +2. **Viac tréningových dát alebo viac krokov automaticky neznamenalo lepší výsledok na každom benchmarku.** + Titan 225k spracoval približne 3,6 milióna príkladov, ale na M-MMLU, HellaSwag a SKLegal bol slabší než viacero 100k Hruška experimentov. + +3. **Titan 225k sa výrazne zlepšil v generatívnom question-answering benchmarku SKQuad oproti Titan 25k.** + F1 stúplo z `25.8678` na `35.0471`. + +4. **Starší Alpaca QLoRA model zostáva silný na TruthfulQA MC1.** + Dosiahol `0.2391`, čo je viac než všetky nové Aya experimenty. To môže znamenať, že rozdielna tréningová dátová distribúcia zachováva alebo posilňuje iné schopnosti modelu. + +5. **Najlepší Hruška validačný loss a najlepší benchmarkový model nie sú totožné.** + Experiment 10 je najlepší podľa Aya eval loss, zatiaľ čo experiment 08 je najvýraznejší podľa širšieho benchmarkového profilu. + + +--- + +## Dôležité obmedzenia porovnania + +Pri interpretácii výsledkov treba zohľadniť: + +- Hruška modely boli trénované na fixnom 100k výbere Aya dát, Titan na streamovanej veľkej časti Aya datasetu, +- konfigurácie tréningu sa medzi Hruškou a Titanom líšili, +- Hruška benchmarky používali `bf16`, Titan `fp16`, +- starší Alpaca benchmark vznikol v inom softvérovom prostredí, +- starý `truthfulqa_sk_mc2` používal inú verziu úlohy, +- nebol zatiaľ zrobený benchmark základného modelu `slovak-nlp/mistral-sk-7b` bez adaptéru. + + diff --git a/lmeval/evaluation/lmeval_custom_tasks/sklegal.yaml b/lmeval/costom/lmeval_custom_tasks/sklegal.yaml similarity index 100% rename from lmeval/evaluation/lmeval_custom_tasks/sklegal.yaml rename to lmeval/costom/lmeval_custom_tasks/sklegal.yaml diff --git a/lmeval/evaluation/lmeval_custom_tasks/skquad.yaml b/lmeval/costom/lmeval_custom_tasks/skquad.yaml similarity index 100% rename from lmeval/evaluation/lmeval_custom_tasks/skquad.yaml rename to lmeval/costom/lmeval_custom_tasks/skquad.yaml diff --git a/lmeval/evaluation/lmeval_custom_tasks/task.py b/lmeval/costom/lmeval_custom_tasks/task.py similarity index 100% rename from lmeval/evaluation/lmeval_custom_tasks/task.py rename to lmeval/costom/lmeval_custom_tasks/task.py diff --git a/lmeval/lmeval_comparison_all.csv b/lmeval/lmeval_comparison_all.csv new file mode 100644 index 0000000..b41f196 --- /dev/null +++ b/lmeval/lmeval_comparison_all.csv @@ -0,0 +1,14 @@ +model,training_eval_loss,arc_acc,arc_acc_norm,hellaswag_acc,hellaswag_acc_norm,m_mmlu_acc,truthfulqa_mc1_acc,truthfulqa_mc2_acc,sklegal_acc,sklegal_acc_norm,skquad_exact,skquad_f1,skquad_HasAns_exact,skquad_HasAns_f1,skquad_NoAns_exact,skquad_NoAns_f1,skquad_best_exact,skquad_best_f1 +alpaca_qlora,,0.32591958939264326,0.3473053892215569,0.4133895624670532,0.5105956773853453,0.2928341754708314,0.2390745501285347,0.39419828418377467,0.2841079460269865,0.46926536731634183,1.9722425127830534,27.7993811825338,2.410565457109886,34.145591726147124,0.05605381165919283,0.05605381165919283,18.61629969738078,28.301192071377013 +01_baseline_r16_a16_lr5e5_adamw,0.7567,0.30196749358426006,0.34046193327630453,0.41560358460727465,0.5209277807063785,0.3549992344204563,0.17994858611825193,0.34391089322085056,0.28785607196401797,0.46851574212893554,14.160492538870917,36.6797346829585,17.38684446723939,45.05730189342111,0.05605381165919283,0.05605381165919283,18.689345716372742,36.915470604294036 +02_optimizer_adamw8bit_r16_a16_lr5e5,0.7521,0.29597946963216426,0.3378956372968349,0.4130732735898788,0.521244069583553,0.35438677078548464,0.1928020565552699,0.35373417730948353,0.2886056971514243,0.4715142428785607,13.76395700720025,36.21782844597198,16.912424669829466,44.50255801997045,0.0,0.0,18.63716998852134,36.46155623142899 +03_rank8_a16_lr5e5,0.7586,0.28999144568006846,0.33105218135158254,0.4137058513442277,0.5167105956773853,0.34550604807839536,0.18380462724935734,0.3424520772406517,0.29760119940029983,0.4715142428785607,13.993530209746426,36.36378811056779,17.194512116938068,44.68190556014504,0.0,0.0,18.77282688093499,36.37507422583225 +04_rank32_a32_lr5e5,0.7292,0.2891360136869119,0.33532934131736525,0.41412756984712706,0.5200843437005799,0.3449701423977951,0.19023136246786632,0.3367013908863517,0.2908545727136432,0.47301349325337333,13.32568089324846,36.02282635500946,16.361071932299012,44.2501275753373,0.05605381165919283,0.05605381165919283,18.67891057080246,36.12444280653584 +05_r16_alpha32_lr5e5,0.7323,0.2994011976047904,0.33361847733105215,0.4130732735898788,0.5190300474433316,0.35714285714285715,0.19665809768637532,0.3555125811399826,0.29685157421289354,0.46701649175412296,14.598768652822708,37.873456685535025,17.925375048083087,46.52408455154278,0.05605381165919283,0.05605381165919283,18.77282688093499,38.068000672246654 +06_lr1e4_r16_a16,0.7184,0.29597946963216426,0.33105218135158254,0.411070110701107,0.5207169214549289,0.3549992344204563,0.19023136246786632,0.34733281230911506,0.2841079460269865,0.46551724137931033,14.014400500886987,37.57962398015711,17.207334273624824,46.16303841541808,0.05605381165919283,0.05605381165919283,18.80413231764583,37.69906094942537 +07_lr2e5_r16_a16,0.7912,0.29426860564585117,0.32763045337895635,0.41560358460727465,0.522509225092251,0.3379268105956209,0.16709511568123395,0.32463255972324384,0.29610194902548725,0.47226386806596704,11.322132943754566,33.83807391933327,13.899217848442108,41.565618972813276,0.05605381165919283,0.05605381165919283,18.61629969738078,33.95862223270098 +08_attention_only_r16_a16_lr5e5,0.8094,0.31565440547476475,0.35243798118049613,0.4157090142329995,0.5217712177121772,0.3448170264890522,0.18766066838046272,0.3673805917101876,0.2893553223388306,0.4767616191904048,18.136283001147866,43.37517960476381,22.28490832157969,53.29713375464183,0.0,0.0,19.628508817698005,43.397808641902465 +09_batch2_accum4_r16_a16_lr5e5,0.7563,0.29597946963216426,0.3378956372968349,0.4136004217185029,0.5209277807063785,0.35140101056499773,0.18637532133676094,0.340221777957301,0.2871064467766117,0.46926536731634183,13.388291766670145,35.3128443275733,16.450827029106296,43.3905612503058,0.0,0.0,18.62673484295106,35.431132341766855 +10_r16_a32_lr1e4,0.6918,0.3028229255774166,0.32934131736526945,0.41012124406958356,0.515656299420137,0.3448170264890522,0.20565552699228792,0.34795284436431645,0.2698650674662669,0.4617691154422789,14.160492538870917,37.75612784500817,17.38684446723939,46.37991705843228,0.05605381165919283,0.05605381165919283,18.77282688093499,38.08192856860334 +titan_aya_25k,,0.2643284858853721,0.32506415739948674,0.4048497627833421,0.4942540853979968,0.3341754708314194,0.17609254498714652,0.32507560661819124,0.2653673163418291,0.4587706146926537,4.633204633204633,25.867828563515147,5.667393255545583,31.759379551758638,0.11210762331838565,0.11210762331838565,18.61629969738078,25.922669866068382 +titan_aya_225k,,0.30025662959794697,0.34473909324208724,0.39768054823405374,0.4792830785450712,0.2719338539274231,0.20051413881748073,0.36134935010431024,0.25337331334332835,0.43928035982008995,10.883856829802776,35.04710724582597,13.373509424285166,43.0640375351648,0.0,0.0,18.64760513409162,35.17783479678552 diff --git a/lmeval/plots/lmeval_main_metrics_chart.pdf b/lmeval/mistral_alpaca(titan)/plots/lmeval_main_metrics_chart.pdf similarity index 100% rename from lmeval/plots/lmeval_main_metrics_chart.pdf rename to lmeval/mistral_alpaca(titan)/plots/lmeval_main_metrics_chart.pdf diff --git a/lmeval/plots/lmeval_main_metrics_chart.png b/lmeval/mistral_alpaca(titan)/plots/lmeval_main_metrics_chart.png similarity index 100% rename from lmeval/plots/lmeval_main_metrics_chart.png rename to lmeval/mistral_alpaca(titan)/plots/lmeval_main_metrics_chart.png diff --git a/lmeval/results/lmeval_results_full_table.csv b/lmeval/mistral_alpaca(titan)/results/lmeval_results_full_table.csv similarity index 100% rename from lmeval/results/lmeval_results_full_table.csv rename to lmeval/mistral_alpaca(titan)/results/lmeval_results_full_table.csv diff --git a/lmeval/mistral_alpaca(titan)/results/results_2026-08-17T21-53-33.966513.json b/lmeval/mistral_alpaca(titan)/results/results_2026-08-17T21-53-33.966513.json new file mode 100644 index 0000000..e49ed05 --- /dev/null +++ b/lmeval/mistral_alpaca(titan)/results/results_2026-08-17T21-53-33.966513.json @@ -0,0 +1,406 @@ +{ + "results": { + "arc_sk": { + "name": "arc_sk", + "alias": "arc_sk", + "sample_len": 1169, + "acc,none": 0.32591958939264326, + "acc_stderr,none": 0.013714805850719354, + "acc_norm,none": 0.3473053892215569, + "acc_norm_stderr,none": 0.013931226499492357 + }, + "hellaswag_sk": { + "name": "hellaswag_sk", + "alias": "hellaswag_sk", + "sample_len": 9485, + "acc,none": 0.4133895624670532, + "acc_stderr,none": 0.0050566031856511765, + "acc_norm,none": 0.5105956773853453, + "acc_norm_stderr,none": 0.00513306418374639 + }, + "m_mmlu_sk": { + "name": "m_mmlu_sk", + "alias": "m_mmlu_sk", + "sample_len": 13062, + "acc,none": 0.2928341754708314, + "acc_stderr,none": 0.003981835345774282 + }, + "truthfulqa_sk_mc1": { + "name": "truthfulqa_sk_mc1", + "alias": "truthfulqa_sk_mc1", + "sample_len": 778, + "acc,none": 0.2390745501285347, + "acc_stderr,none": 0.015301260856408238 + }, + "truthfulqa_sk_mc2": { + "name": "truthfulqa_sk_mc2", + "alias": "truthfulqa_sk_mc2", + "sample_len": 778, + "acc,none": 0.39419828418377467, + "acc_stderr,none": 0.014881741560729694 + } + }, + "group_subtasks": {}, + "configs": { + "arc_sk": { + "task": "arc_sk", + "dataset_path": "alexandrainst/m_arc", + "dataset_name": "sk", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n # breakpoint()\n out_doc = {\n \"id\": doc[\"id\"],\n \"query\": \"Question: \" + preprocess(doc[\"instruction\"]) + \"\\nAnswer:\",\n \"choices\": [\n preprocess(option)\n for option in [\n doc[\"option_a\"],\n doc[\"option_b\"],\n doc[\"option_c\"],\n doc[\"option_d\"],\n doc[\"option_e\"],\n ]\n if option\n ],\n \"gold\": [\"A\", \"B\", \"C\", \"D\", \"E\"].index(doc[\"answer\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "gold", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": "", + "fewshot_indices": null, + "samples": null, + "doc_to_text": "query", + "doc_to_choice": "choices", + "doc_to_target": "gold", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "query", + "metadata": { + "version": 2.0, + "pretrained": "slovak-nlp/mistral-sk-7b", + "peft": "Jakub1320/mistral-sk-7b-slovak-alpaca-qlora", + "dtype": "float16", + "config_source": "/home/schwarc/miniconda3/envs/llama_factory/lib/python3.11/site-packages/lm_eval/tasks/okapi/arc_multilingual/arc_sk.yaml" + } + }, + "hellaswag_sk": { + "task": "hellaswag_sk", + "dataset_path": "alexandrainst/m_hellaswag", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "{{label.lstrip()}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": "", + "fewshot_indices": null, + "samples": null, + "doc_to_text": "query", + "doc_to_choice": "choices", + "doc_to_target": "{{label.lstrip()}}", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0, + "pretrained": "slovak-nlp/mistral-sk-7b", + "peft": "Jakub1320/mistral-sk-7b-slovak-alpaca-qlora", + "dtype": "float16", + "config_source": "/home/schwarc/miniconda3/envs/llama_factory/lib/python3.11/site-packages/lm_eval/tasks/okapi/hellaswag_multilingual/hellaswag_sk.yaml" + } + }, + "m_mmlu_sk": { + "task": "m_mmlu_sk", + "dataset_path": "alexandrainst/m_mmlu", + "dataset_name": "sk", + "test_split": "test", + "fewshot_split": "train", + "doc_to_text": "{{instruction.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:", + "doc_to_target": "answer", + "unsafe_code": false, + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n", + "split": "train", + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{instruction.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:", + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "doc_to_target": "answer", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0, + "pretrained": "slovak-nlp/mistral-sk-7b", + "peft": "Jakub1320/mistral-sk-7b-slovak-alpaca-qlora", + "dtype": "float16", + "config_source": "/home/schwarc/miniconda3/envs/llama_factory/lib/python3.11/site-packages/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_sk.yaml" + } + }, + "truthfulqa_sk_mc1": { + "task": "truthfulqa_sk_mc1", + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc1_choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": "", + "fewshot_indices": null, + "samples": null, + "doc_to_text": "query", + "doc_to_choice": "mc1_choices", + "doc_to_target": 0, + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0, + "pretrained": "slovak-nlp/mistral-sk-7b", + "peft": "Jakub1320/mistral-sk-7b-slovak-alpaca-qlora", + "dtype": "float16", + "config_source": "/home/schwarc/miniconda3/envs/llama_factory/lib/python3.11/site-packages/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sk_mc1.yaml" + } + }, + "truthfulqa_sk_mc2": { + "task": "truthfulqa_sk_mc2", + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc2_choices", + "process_results": "def process_results_mc2(doc, results):\n ll, _ = zip(*results)\n ll = np.array(ll)\n\n # Convert log-likelihoods to probabilities.\n probs = np.exp(ll)\n\n # Normalize probabilities.\n probs_norm = probs / np.sum(probs)\n\n labels = np.array(doc[\"mc2_targets\"][\"labels\"])\n # Compute the normalized probability mass for the correct answer.\n pm_true = np.sum(probs_norm[labels == 1])\n\n return {\"acc\": pm_true}\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": "", + "fewshot_indices": null, + "samples": null, + "doc_to_text": "query", + "doc_to_choice": "mc2_choices", + "doc_to_target": 0, + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 2.0, + "pretrained": "slovak-nlp/mistral-sk-7b", + "peft": "Jakub1320/mistral-sk-7b-slovak-alpaca-qlora", + "dtype": "float16", + "config_source": "/home/schwarc/miniconda3/envs/llama_factory/lib/python3.11/site-packages/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sk_mc2.yaml" + } + } + }, + "versions": { + "arc_sk": 2.0, + "hellaswag_sk": 1.0, + "m_mmlu_sk": 0.0, + "truthfulqa_sk_mc1": 1.0, + "truthfulqa_sk_mc2": 2.0 + }, + "n-shot": { + "arc_sk": 0, + "hellaswag_sk": 0, + "m_mmlu_sk": 0, + "truthfulqa_sk_mc1": 0, + "truthfulqa_sk_mc2": 0 + }, + "higher_is_better": { + "arc_sk": { + "acc": true, + "acc_norm": true + }, + "hellaswag_sk": { + "acc": true, + "acc_norm": true + }, + "m_mmlu_sk": { + "acc": true + }, + "truthfulqa_sk_mc1": { + "acc": true + }, + "truthfulqa_sk_mc2": { + "acc": true + } + }, + "n-samples": { + "arc_sk": { + "original": 1169, + "effective": 1169 + }, + "hellaswag_sk": { + "original": 9485, + "effective": 9485 + }, + "m_mmlu_sk": { + "original": 13062, + "effective": 13062 + }, + "truthfulqa_sk_mc1": { + "original": 778, + "effective": 778 + }, + "truthfulqa_sk_mc2": { + "original": 778, + "effective": 778 + } + }, + "config": { + "model": "hf", + "model_args": { + "pretrained": "slovak-nlp/mistral-sk-7b", + "peft": "Jakub1320/mistral-sk-7b-slovak-alpaca-qlora", + "dtype": "float16" + }, + "model_num_parameters": 7283675136, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "089497ae0d72018e9591895c54f774bf0b4a83a4", + "peft_sha": "b7d5ba53c81f33334442196915712b5b028d4005", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": {}, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": "53504a96", + "date": 1786990597.4402645, + "pretty_env_info": "PyTorch version: 2.6.0+cu124\nIs debug build: False\nCUDA used to build PyTorch: 12.4\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 20.04.6 LTS (x86_64)\nGCC version: (Ubuntu 9.4.0-1ubuntu1~20.04.2) 9.4.0\nClang version: Could not collect\nCMake version: version 3.16.3\nLibc version: glibc-2.31\n\nPython version: 3.11.15 (main, Mar 11 2026, 17:20:07) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-5.15.0-119-generic-x86_64-with-glibc2.31\nIs CUDA available: True\nCUDA runtime version: 10.1.243\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA TITAN RTX\nGPU 1: NVIDIA TITAN RTX\n\nNvidia driver version: 575.57.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.10.2\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 20\nOn-line CPU(s) list: 0-19\nThread(s) per core: 2\nCore(s) per socket: 10\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Core(TM) i9-9820X CPU @ 3.30GHz\nStepping: 4\nCPU MHz: 3300.000\nCPU max MHz: 4200,0000\nCPU min MHz: 1200,0000\nBogoMIPS: 6599.98\nL1d cache: 320 KiB\nL1i cache: 320 KiB\nL2 cache: 10 MiB\nL3 cache: 16,5 MiB\nNUMA node0 CPU(s): 0-19\nVulnerability Gather data sampling: Mitigation; Microcode\nVulnerability Itlb multihit: KVM: Mitigation: VMX unsupported\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Mitigation; Clear CPU buffers; SMT vulnerable\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Mmio stale data: Mitigation; Clear CPU buffers; SMT vulnerable\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; IBRS; IBPB conditional; STIBP conditional; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Mitigation; Clear CPU buffers; SMT vulnerable\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 invpcid_single pti ssbd mba ibrs ibpb stibp fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm cqm mpx rdt_a avx512f avx512dq rdseed adx smap clflushopt clwb intel_pt avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req md_clear flush_l1d arch_capabilities\n\nVersions of relevant libraries:\n[pip3] numpy==2.4.6\n[pip3] nvidia-cublas==13.1.1.3\n[pip3] nvidia-cublas-cu12==12.4.5.8\n[pip3] nvidia-cuda-cupti==13.0.85\n[pip3] nvidia-cuda-cupti-cu12==12.4.127\n[pip3] nvidia-cuda-nvrtc==13.0.88\n[pip3] nvidia-cuda-nvrtc-cu12==12.4.127\n[pip3] nvidia-cuda-runtime==13.0.96\n[pip3] nvidia-cuda-runtime-cu12==12.4.127\n[pip3] nvidia-cudnn-cu12==9.1.0.70\n[pip3] nvidia-cudnn-cu13==9.20.0.48\n[pip3] nvidia-cufft==12.0.0.61\n[pip3] nvidia-cufft-cu12==11.2.1.3\n[pip3] nvidia-curand==10.4.0.35\n[pip3] nvidia-curand-cu12==10.3.5.147\n[pip3] nvidia-cusolver==12.0.4.66\n[pip3] nvidia-cusolver-cu12==11.6.1.9\n[pip3] nvidia-cusparse==12.6.3.3\n[pip3] nvidia-cusparse-cu12==12.3.1.170\n[pip3] nvidia-cusparselt-cu12==0.6.2\n[pip3] nvidia-cusparselt-cu13==0.8.1\n[pip3] nvidia-nccl-cu12==2.21.5\n[pip3] nvidia-nccl-cu13==2.29.7\n[pip3] nvidia-nvjitlink==13.0.88\n[pip3] nvidia-nvjitlink-cu12==12.4.127\n[pip3] nvidia-nvtx==13.0.85\n[pip3] nvidia-nvtx-cu12==12.4.127\n[pip3] torch==2.6.0+cu124\n[pip3] torchaudio==2.6.0+cu124\n[pip3] torchdata==0.11.0\n[pip3] torchvision==0.21.0+cu124\n[pip3] triton==3.2.0\n[conda] numpy 2.4.6 pypi_0 pypi\n[conda] nvidia-cublas 13.1.1.3 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.4.5.8 pypi_0 pypi\n[conda] nvidia-cuda-cupti 13.0.85 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc 13.0.88 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-runtime 13.0.96 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.1.0.70 pypi_0 pypi\n[conda] nvidia-cudnn-cu13 9.20.0.48 pypi_0 pypi\n[conda] nvidia-cufft 12.0.0.61 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.2.1.3 pypi_0 pypi\n[conda] nvidia-curand 10.4.0.35 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.5.147 pypi_0 pypi\n[conda] nvidia-cusolver 12.0.4.66 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.6.1.9 pypi_0 pypi\n[conda] nvidia-cusparse 12.6.3.3 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.3.1.170 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.6.2 pypi_0 pypi\n[conda] nvidia-cusparselt-cu13 0.8.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.21.5 pypi_0 pypi\n[conda] nvidia-nccl-cu13 2.29.7 pypi_0 pypi\n[conda] nvidia-nvjitlink 13.0.88 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-nvtx 13.0.85 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.4.127 pypi_0 pypi\n[conda] torch 2.6.0+cu124 pypi_0 pypi\n[conda] torchaudio 2.6.0+cu124 pypi_0 pypi\n[conda] torchdata 0.11.0 pypi_0 pypi\n[conda] torchvision 0.21.0+cu124 pypi_0 pypi\n[conda] triton 3.2.0 pypi_0 pypi", + "transformers_version": "5.5.0", + "lm_eval_version": "0.4.12", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "", + "2" + ], + "tokenizer_eos_token": [ + "", + "2" + ], + "tokenizer_bos_token": [ + "", + "1" + ], + "eot_token_id": 2, + "max_length": 32768, + "task_hashes": {}, + "model_source": "hf", + "model_name": "Jakub1320/mistral-sk-7b-slovak-alpaca-qlora", + "model_name_sanitized": "Jakub1320__mistral-sk-7b-slovak-alpaca-qlora", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": null, + "chat_template": null, + "chat_template_sha": null, + "total_evaluation_time_seconds": "5825.593601043802" +} \ No newline at end of file diff --git a/lmeval/mistral_alpaca(titan)/results/results_2026-08-18T14-34-57.786558.json b/lmeval/mistral_alpaca(titan)/results/results_2026-08-18T14-34-57.786558.json new file mode 100644 index 0000000..ffcff4c --- /dev/null +++ b/lmeval/mistral_alpaca(titan)/results/results_2026-08-18T14-34-57.786558.json @@ -0,0 +1,208 @@ +{ + "results": { + "sklegal": { + "name": "sklegal", + "alias": "sklegal", + "sample_len": 1334, + "acc,none": 0.2841079460269865, + "acc_stderr,none": 0.012352372689133341, + "acc_norm,none": 0.46926536731634183, + "acc_norm_stderr,none": 0.013668878751636108 + }, + "skquad": { + "name": "skquad", + "alias": "skquad", + "sample_len": 9583, + "exact,none": 1.9722425127830534, + "exact_stderr,none": "N/A", + "f1,none": 27.7993811825338, + "f1_stderr,none": "N/A", + "HasAns_exact,none": 2.410565457109886, + "HasAns_exact_stderr,none": "N/A", + "HasAns_f1,none": 34.145591726147124, + "HasAns_f1_stderr,none": "N/A", + "NoAns_exact,none": 0.05605381165919283, + "NoAns_exact_stderr,none": "N/A", + "NoAns_f1,none": 0.05605381165919283, + "NoAns_f1_stderr,none": "N/A", + "best_exact,none": 18.61629969738078, + "best_exact_stderr,none": "N/A", + "best_f1,none": 28.301192071377013, + "best_f1_stderr,none": "N/A" + } + }, + "group_subtasks": {}, + "configs": { + "sklegal": { + "task": "sklegal", + "dataset_path": "TUKE-KEMT/slovak-legal-exam", + "test_split": "test", + "doc_to_text": "{{prompt}} ", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{answers}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": "{{prompt}} ", + "doc_to_choice": "{{answers}}", + "doc_to_target": "label", + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1, + "pretrained": "slovak-nlp/mistral-sk-7b", + "peft": "Jakub1320/mistral-sk-7b-slovak-alpaca-qlora", + "dtype": "float16", + "config_source": "/home/schwarc/diplomovka/lmeval_missing_tasks/sklegal.yaml" + } + }, + "skquad": { + "task": "skquad", + "unsafe_code": false, + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "default", + "split": null, + "process_docs": null, + "fewshot_indices": null, + "samples": null, + "doc_to_text": null, + "doc_to_choice": null, + "doc_to_target": null, + "gen_prefix": null, + "fewshot_delimiter": "\n\n", + "target_delimiter": " " + }, + "num_fewshot": 0, + "output_type": "generate_until", + "generation_kwargs": { + "temperature": 0.0, + "do_sample": false, + "max_gen_toks": 256, + "until": [ + "\n\n" + ] + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 3 + } + } + }, + "versions": { + "sklegal": 1, + "skquad": 3 + }, + "n-shot": { + "sklegal": 0, + "skquad": 0 + }, + "higher_is_better": { + "sklegal": { + "acc": true, + "acc_norm": true + }, + "skquad": { + "exact": true, + "f1": true, + "HasAns_exact": true, + "HasAns_f1": true, + "NoAns_exact": true, + "NoAns_f1": true, + "best_exact": true, + "best_f1": true + } + }, + "n-samples": { + "sklegal": { + "original": 1334, + "effective": 1334 + }, + "skquad": { + "original": 9583, + "effective": 9583 + } + }, + "config": { + "model": "hf", + "model_args": { + "pretrained": "slovak-nlp/mistral-sk-7b", + "peft": "Jakub1320/mistral-sk-7b-slovak-alpaca-qlora", + "dtype": "float16" + }, + "model_num_parameters": 7283675136, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "089497ae0d72018e9591895c54f774bf0b4a83a4", + "peft_sha": "b7d5ba53c81f33334442196915712b5b028d4005", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": {}, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": null, + "date": 1786998254.1614816, + "pretty_env_info": "PyTorch version: 2.6.0+cu124\nIs debug build: False\nCUDA used to build PyTorch: 12.4\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 20.04.6 LTS (x86_64)\nGCC version: (Ubuntu 9.4.0-1ubuntu1~20.04.2) 9.4.0\nClang version: Could not collect\nCMake version: version 3.16.3\nLibc version: glibc-2.31\n\nPython version: 3.11.15 (main, Mar 11 2026, 17:20:07) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-5.15.0-119-generic-x86_64-with-glibc2.31\nIs CUDA available: True\nCUDA runtime version: 10.1.243\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA TITAN RTX\nGPU 1: NVIDIA TITAN RTX\n\nNvidia driver version: 575.57.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.10.2\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 20\nOn-line CPU(s) list: 0-19\nThread(s) per core: 2\nCore(s) per socket: 10\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Core(TM) i9-9820X CPU @ 3.30GHz\nStepping: 4\nCPU MHz: 3300.000\nCPU max MHz: 4200,0000\nCPU min MHz: 1200,0000\nBogoMIPS: 6599.98\nL1d cache: 320 KiB\nL1i cache: 320 KiB\nL2 cache: 10 MiB\nL3 cache: 16,5 MiB\nNUMA node0 CPU(s): 0-19\nVulnerability Gather data sampling: Mitigation; Microcode\nVulnerability Itlb multihit: KVM: Mitigation: VMX unsupported\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Mitigation; Clear CPU buffers; SMT vulnerable\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Mmio stale data: Mitigation; Clear CPU buffers; SMT vulnerable\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; IBRS; IBPB conditional; STIBP conditional; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Mitigation; Clear CPU buffers; SMT vulnerable\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 invpcid_single pti ssbd mba ibrs ibpb stibp fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm cqm mpx rdt_a avx512f avx512dq rdseed adx smap clflushopt clwb intel_pt avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req md_clear flush_l1d arch_capabilities\n\nVersions of relevant libraries:\n[pip3] numpy==2.4.6\n[pip3] nvidia-cublas==13.1.1.3\n[pip3] nvidia-cublas-cu12==12.4.5.8\n[pip3] nvidia-cuda-cupti==13.0.85\n[pip3] nvidia-cuda-cupti-cu12==12.4.127\n[pip3] nvidia-cuda-nvrtc==13.0.88\n[pip3] nvidia-cuda-nvrtc-cu12==12.4.127\n[pip3] nvidia-cuda-runtime==13.0.96\n[pip3] nvidia-cuda-runtime-cu12==12.4.127\n[pip3] nvidia-cudnn-cu12==9.1.0.70\n[pip3] nvidia-cudnn-cu13==9.20.0.48\n[pip3] nvidia-cufft==12.0.0.61\n[pip3] nvidia-cufft-cu12==11.2.1.3\n[pip3] nvidia-curand==10.4.0.35\n[pip3] nvidia-curand-cu12==10.3.5.147\n[pip3] nvidia-cusolver==12.0.4.66\n[pip3] nvidia-cusolver-cu12==11.6.1.9\n[pip3] nvidia-cusparse==12.6.3.3\n[pip3] nvidia-cusparse-cu12==12.3.1.170\n[pip3] nvidia-cusparselt-cu12==0.6.2\n[pip3] nvidia-cusparselt-cu13==0.8.1\n[pip3] nvidia-nccl-cu12==2.21.5\n[pip3] nvidia-nccl-cu13==2.29.7\n[pip3] nvidia-nvjitlink==13.0.88\n[pip3] nvidia-nvjitlink-cu12==12.4.127\n[pip3] nvidia-nvtx==13.0.85\n[pip3] nvidia-nvtx-cu12==12.4.127\n[pip3] torch==2.6.0+cu124\n[pip3] torchaudio==2.6.0+cu124\n[pip3] torchdata==0.11.0\n[pip3] torchvision==0.21.0+cu124\n[pip3] triton==3.2.0\n[conda] numpy 2.4.6 pypi_0 pypi\n[conda] nvidia-cublas 13.1.1.3 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.4.5.8 pypi_0 pypi\n[conda] nvidia-cuda-cupti 13.0.85 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc 13.0.88 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-runtime 13.0.96 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.1.0.70 pypi_0 pypi\n[conda] nvidia-cudnn-cu13 9.20.0.48 pypi_0 pypi\n[conda] nvidia-cufft 12.0.0.61 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.2.1.3 pypi_0 pypi\n[conda] nvidia-curand 10.4.0.35 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.5.147 pypi_0 pypi\n[conda] nvidia-cusolver 12.0.4.66 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.6.1.9 pypi_0 pypi\n[conda] nvidia-cusparse 12.6.3.3 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.3.1.170 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.6.2 pypi_0 pypi\n[conda] nvidia-cusparselt-cu13 0.8.1 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.21.5 pypi_0 pypi\n[conda] nvidia-nccl-cu13 2.29.7 pypi_0 pypi\n[conda] nvidia-nvjitlink 13.0.88 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-nvtx 13.0.85 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.4.127 pypi_0 pypi\n[conda] torch 2.6.0+cu124 pypi_0 pypi\n[conda] torchaudio 2.6.0+cu124 pypi_0 pypi\n[conda] torchdata 0.11.0 pypi_0 pypi\n[conda] torchvision 0.21.0+cu124 pypi_0 pypi\n[conda] triton 3.2.0 pypi_0 pypi", + "transformers_version": "5.5.0", + "lm_eval_version": "0.4.12", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "", + "2" + ], + "tokenizer_eos_token": [ + "", + "2" + ], + "tokenizer_bos_token": [ + "", + "1" + ], + "eot_token_id": 2, + "max_length": 32768, + "task_hashes": {}, + "model_source": "hf", + "model_name": "Jakub1320/mistral-sk-7b-slovak-alpaca-qlora", + "model_name_sanitized": "Jakub1320__mistral-sk-7b-slovak-alpaca-qlora", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": null, + "chat_template": null, + "chat_template_sha": null, + "total_evaluation_time_seconds": "58250.00474518072" +} \ No newline at end of file diff --git a/lmeval/mistral_aya(hruska)/01_baseline_r16_a16_lr5e5_adamw/results_2026-09-30T21-23-33.435952.json b/lmeval/mistral_aya(hruska)/01_baseline_r16_a16_lr5e5_adamw/results_2026-09-30T21-23-33.435952.json new file mode 100644 index 0000000..9d3e2d6 --- /dev/null +++ b/lmeval/mistral_aya(hruska)/01_baseline_r16_a16_lr5e5_adamw/results_2026-09-30T21-23-33.435952.json @@ -0,0 +1,432 @@ +{ + "results": { + "arc_sk": { + "alias": "arc_sk", + "acc,none": 0.30196749358426006, + "acc_stderr,none": 0.013433726418301333, + "acc_norm,none": 0.34046193327630453, + "acc_norm_stderr,none": 0.013865412436810633 + }, + "hellaswag_sk": { + "alias": "hellaswag_sk", + "acc,none": 0.41560358460727465, + "acc_stderr,none": 0.0050605490817228595, + "acc_norm,none": 0.5209277807063785, + "acc_norm_stderr,none": 0.005129717875513435 + }, + "m_mmlu_sk": { + "alias": "m_mmlu_sk", + "acc,none": 0.3549992344204563, + "acc_stderr,none": 0.004187025526122447 + }, + "sklegal": { + "alias": "sklegal", + "acc,none": 0.28785607196401797, + "acc_stderr,none": 0.012400994375995684, + "acc_norm,none": 0.46851574212893554, + "acc_norm_stderr,none": 0.013667598799495391 + }, + "skquad": { + "alias": "skquad", + "exact,none": 14.160492538870917, + "exact_stderr,none": "N/A", + "f1,none": 36.6797346829585, + "f1_stderr,none": "N/A", + "HasAns_exact,none": 17.38684446723939, + "HasAns_exact_stderr,none": "N/A", + "HasAns_f1,none": 45.05730189342111, + "HasAns_f1_stderr,none": "N/A", + "NoAns_exact,none": 0.05605381165919283, + "NoAns_exact_stderr,none": "N/A", + "NoAns_f1,none": 0.05605381165919283, + "NoAns_f1_stderr,none": "N/A", + "best_exact,none": 18.689345716372742, + "best_exact_stderr,none": "N/A", + "best_f1,none": 36.915470604294036, + "best_f1_stderr,none": "N/A" + }, + "truthfulqa_sk_mc1": { + "alias": "truthfulqa_sk_mc1", + "acc,none": 0.17994858611825193, + "acc_stderr,none": 0.013781111636253436 + }, + "truthfulqa_sk_mc2": { + "alias": "truthfulqa_sk_mc2", + "acc,none": 0.34391089322085056, + "acc_stderr,none": 0.014436484326919677 + } + }, + "group_subtasks": { + "arc_sk": [], + "hellaswag_sk": [], + "m_mmlu_sk": [], + "sklegal": [], + "skquad": [], + "truthfulqa_sk_mc1": [], + "truthfulqa_sk_mc2": [] + }, + "configs": { + "arc_sk": { + "task": "arc_sk", + "tag": [ + "arc_multilingual" + ], + "dataset_path": "alexandrainst/m_arc", + "dataset_name": "sk", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n # breakpoint()\n out_doc = {\n \"id\": doc[\"id\"],\n \"query\": \"Question: \" + preprocess(doc[\"instruction\"]) + \"\\nAnswer:\",\n \"choices\": [\n preprocess(option)\n for option in [\n doc[\"option_a\"],\n doc[\"option_b\"],\n doc[\"option_c\"],\n doc[\"option_d\"],\n doc[\"option_e\"],\n ]\n if option\n ],\n \"gold\": [\"A\", \"B\", \"C\", \"D\", \"E\"].index(doc[\"answer\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "gold", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "query", + "metadata": { + "version": 2.0 + } + }, + "hellaswag_sk": { + "task": "hellaswag_sk", + "tag": [ + "hellaswag_multilingual" + ], + "dataset_path": "alexandrainst/m_hellaswag", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "{{label.lstrip()}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0 + } + }, + "m_mmlu_sk": { + "task": "m_mmlu_sk", + "tag": [ + "m_mmlu" + ], + "dataset_path": "alexandrainst/m_mmlu", + "dataset_name": "sk", + "test_split": "test", + "fewshot_split": "train", + "doc_to_text": "{{instruction.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:", + "doc_to_target": "answer", + "unsafe_code": false, + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n" + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "sklegal": { + "task": "sklegal", + "dataset_path": "TUKE-KEMT/slovak-legal-exam", + "test_split": "test", + "doc_to_text": "{{prompt}} ", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{answers}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1 + } + }, + "skquad": { + "task": "skquad", + "unsafe_code": false, + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "output_type": "generate_until", + "generation_kwargs": { + "until": [ + "\n\n" + ], + "do_sample": false + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 3 + } + }, + "truthfulqa_sk_mc1": { + "task": "truthfulqa_sk_mc1", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc1_choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + }, + "truthfulqa_sk_mc2": { + "task": "truthfulqa_sk_mc2", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc2_choices", + "process_results": "def process_results_mc2(doc, results):\n lls, is_greedy = zip(*results)\n\n # Split on the first `0` as everything before it is true (`1`).\n split_idx = list(doc[\"mc2_targets\"][\"labels\"]).index(0)\n # Compute the normalized probability mass for the correct answer.\n ll_true, ll_false = lls[:split_idx], lls[split_idx:]\n p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false))\n p_true = p_true / (sum(p_true) + sum(p_false))\n\n return {\"acc\": sum(p_true)}\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + } + }, + "versions": { + "arc_sk": 2.0, + "hellaswag_sk": 1.0, + "m_mmlu_sk": 0.0, + "sklegal": 1, + "skquad": 3, + "truthfulqa_sk_mc1": 1.0, + "truthfulqa_sk_mc2": 1.0 + }, + "n-shot": { + "arc_sk": 0, + "hellaswag_sk": 0, + "m_mmlu_sk": 0, + "sklegal": 0, + "skquad": 0, + "truthfulqa_sk_mc1": 0, + "truthfulqa_sk_mc2": 0 + }, + "higher_is_better": { + "arc_sk": { + "acc": true, + "acc_norm": true + }, + "hellaswag_sk": { + "acc": true, + "acc_norm": true + }, + "m_mmlu_sk": { + "acc": true + }, + "sklegal": { + "acc": true, + "acc_norm": true + }, + "skquad": { + "exact": true, + "f1": true, + "HasAns_exact": true, + "HasAns_f1": true, + "NoAns_exact": true, + "NoAns_f1": true, + "best_exact": true, + "best_f1": true + }, + "truthfulqa_sk_mc1": { + "acc": true + }, + "truthfulqa_sk_mc2": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_sk_mc2": { + "original": 778, + "effective": 778 + }, + "truthfulqa_sk_mc1": { + "original": 778, + "effective": 778 + }, + "skquad": { + "original": 9583, + "effective": 9583 + }, + "sklegal": { + "original": 1334, + "effective": 1334 + }, + "m_mmlu_sk": { + "original": 13062, + "effective": 13062 + }, + "hellaswag_sk": { + "original": 9485, + "effective": 9485 + }, + "arc_sk": { + "original": 1169, + "effective": 1169 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_01_baseline_r16_a16_lr5e5_adamw,dtype=bfloat16", + "model_num_parameters": 7283675136, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "089497ae0d72018e9591895c54f774bf0b4a83a4", + "peft_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": "6d2abda4", + "date": 1790780066.5435824, + "pretty_env_info": "PyTorch version: 2.6.0+cu124\nIs debug build: False\nCUDA used to build PyTorch: 12.4\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.11.16 (main, Aug 27 2026, 14:44:21) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.8.0-138-generic-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA RTX A4500\nGPU 1: NVIDIA RTX A4500\n\nNvidia driver version: 550.163.01\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 16\nOn-line CPU(s) list: 0-15\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) W-2245 CPU @ 3.90GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 8\nSocket(s): 1\nStepping: 7\nCPU(s) scaling MHz: 28%\nCPU max MHz: 4700.0000\nCPU min MHz: 1200.0000\nBogoMIPS: 7799.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 intel_ppin ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm mpx rdt_a avx512f avx512dq rdseed adx smap clflushopt clwb intel_pt avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512_vnni md_clear flush_l1d arch_capabilities ibpb_exit_to_user\nVirtualization: VT-x\nL1d cache: 256 KiB (8 instances)\nL1i cache: 256 KiB (8 instances)\nL2 cache: 8 MiB (8 instances)\nL3 cache: 16.5 MiB (1 instance)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-15\nVulnerability Gather data sampling: Mitigation; Microcode\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX disabled\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Mitigation; Clear CPU buffers; SMT vulnerable\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced / Automatic IBRS; IBPB conditional; PBRSB-eIBRS SW sequence; BHI SW loop, KVM SW loop\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Mitigation; TSX disabled\nVulnerability Vmscape: Mitigation; IBPB before exit to userspace\n\nVersions of relevant libraries:\n[pip3] numpy==2.4.6\n[pip3] nvidia-cublas-cu12==12.4.5.8\n[pip3] nvidia-cuda-cupti-cu12==12.4.127\n[pip3] nvidia-cuda-nvrtc-cu12==12.4.127\n[pip3] nvidia-cuda-runtime-cu12==12.4.127\n[pip3] nvidia-cudnn-cu12==9.1.0.70\n[pip3] nvidia-cufft-cu12==11.2.1.3\n[pip3] nvidia-curand-cu12==10.3.5.147\n[pip3] nvidia-cusolver-cu12==11.6.1.9\n[pip3] nvidia-cusparse-cu12==12.3.1.170\n[pip3] nvidia-cusparselt-cu12==0.6.2\n[pip3] nvidia-nccl-cu12==2.21.5\n[pip3] nvidia-nvjitlink-cu12==12.4.127\n[pip3] nvidia-nvtx-cu12==12.4.127\n[pip3] torch==2.6.0+cu124\n[pip3] triton==3.2.0\n[conda] numpy 2.4.6 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.4.5.8 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.1.0.70 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.2.1.3 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.5.147 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.6.1.9 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.3.1.170 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.6.2 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.21.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.4.127 pypi_0 pypi\n[conda] torch 2.6.0+cu124 pypi_0 pypi\n[conda] triton 3.2.0 pypi_0 pypi", + "transformers_version": "4.48.2", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "", + "2" + ], + "tokenizer_eos_token": [ + "", + "2" + ], + "tokenizer_bos_token": [ + "", + "1" + ], + "eot_token_id": 2, + "max_length": 32768, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/home/schwarc/diplomovka/hruska_compare9_100k_fixed_01_baseline_r16_a16_lr5e5_adamw", + "model_name_sanitized": "__home__schwarc__diplomovka__hruska_compare9_100k_fixed_01_baseline_r16_a16_lr5e5_adamw", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 2617100.761775684, + "end_time": 2640455.236126951, + "total_evaluation_time_seconds": "23354.474351266865" +} \ No newline at end of file diff --git a/lmeval/mistral_aya(hruska)/02_optimizer_adamw8bit_r16_a16_lr5e5/results_2026-10-01T04-23-23.791749.json b/lmeval/mistral_aya(hruska)/02_optimizer_adamw8bit_r16_a16_lr5e5/results_2026-10-01T04-23-23.791749.json new file mode 100644 index 0000000..16fca14 --- /dev/null +++ b/lmeval/mistral_aya(hruska)/02_optimizer_adamw8bit_r16_a16_lr5e5/results_2026-10-01T04-23-23.791749.json @@ -0,0 +1,432 @@ +{ + "results": { + "arc_sk": { + "alias": "arc_sk", + "acc,none": 0.29597946963216426, + "acc_stderr,none": 0.013356788048643698, + "acc_norm,none": 0.3378956372968349, + "acc_norm_stderr,none": 0.013839904571755934 + }, + "hellaswag_sk": { + "alias": "hellaswag_sk", + "acc,none": 0.4130732735898788, + "acc_stderr,none": 0.0050560308898267535, + "acc_norm,none": 0.521244069583553, + "acc_norm_stderr,none": 0.005129580788170514 + }, + "m_mmlu_sk": { + "alias": "m_mmlu_sk", + "acc,none": 0.35438677078548464, + "acc_stderr,none": 0.004185397845395421 + }, + "sklegal": { + "alias": "sklegal", + "acc,none": 0.2886056971514243, + "acc_stderr,none": 0.012410593952463871, + "acc_norm,none": 0.4715142428785607, + "acc_norm_stderr,none": 0.013672532895182896 + }, + "skquad": { + "alias": "skquad", + "exact,none": 13.76395700720025, + "exact_stderr,none": "N/A", + "f1,none": 36.21782844597198, + "f1_stderr,none": "N/A", + "HasAns_exact,none": 16.912424669829466, + "HasAns_exact_stderr,none": "N/A", + "HasAns_f1,none": 44.50255801997045, + "HasAns_f1_stderr,none": "N/A", + "NoAns_exact,none": 0.0, + "NoAns_exact_stderr,none": "N/A", + "NoAns_f1,none": 0.0, + "NoAns_f1_stderr,none": "N/A", + "best_exact,none": 18.63716998852134, + "best_exact_stderr,none": "N/A", + "best_f1,none": 36.46155623142899, + "best_f1_stderr,none": "N/A" + }, + "truthfulqa_sk_mc1": { + "alias": "truthfulqa_sk_mc1", + "acc,none": 0.1928020565552699, + "acc_stderr,none": 0.014152571087810484 + }, + "truthfulqa_sk_mc2": { + "alias": "truthfulqa_sk_mc2", + "acc,none": 0.35373417730948353, + "acc_stderr,none": 0.01457245627897228 + } + }, + "group_subtasks": { + "arc_sk": [], + "hellaswag_sk": [], + "m_mmlu_sk": [], + "sklegal": [], + "skquad": [], + "truthfulqa_sk_mc1": [], + "truthfulqa_sk_mc2": [] + }, + "configs": { + "arc_sk": { + "task": "arc_sk", + "tag": [ + "arc_multilingual" + ], + "dataset_path": "alexandrainst/m_arc", + "dataset_name": "sk", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n # breakpoint()\n out_doc = {\n \"id\": doc[\"id\"],\n \"query\": \"Question: \" + preprocess(doc[\"instruction\"]) + \"\\nAnswer:\",\n \"choices\": [\n preprocess(option)\n for option in [\n doc[\"option_a\"],\n doc[\"option_b\"],\n doc[\"option_c\"],\n doc[\"option_d\"],\n doc[\"option_e\"],\n ]\n if option\n ],\n \"gold\": [\"A\", \"B\", \"C\", \"D\", \"E\"].index(doc[\"answer\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "gold", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "query", + "metadata": { + "version": 2.0 + } + }, + "hellaswag_sk": { + "task": "hellaswag_sk", + "tag": [ + "hellaswag_multilingual" + ], + "dataset_path": "alexandrainst/m_hellaswag", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "{{label.lstrip()}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0 + } + }, + "m_mmlu_sk": { + "task": "m_mmlu_sk", + "tag": [ + "m_mmlu" + ], + "dataset_path": "alexandrainst/m_mmlu", + "dataset_name": "sk", + "test_split": "test", + "fewshot_split": "train", + "doc_to_text": "{{instruction.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:", + "doc_to_target": "answer", + "unsafe_code": false, + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n" + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "sklegal": { + "task": "sklegal", + "dataset_path": "TUKE-KEMT/slovak-legal-exam", + "test_split": "test", + "doc_to_text": "{{prompt}} ", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{answers}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1 + } + }, + "skquad": { + "task": "skquad", + "unsafe_code": false, + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "output_type": "generate_until", + "generation_kwargs": { + "until": [ + "\n\n" + ], + "do_sample": false + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 3 + } + }, + "truthfulqa_sk_mc1": { + "task": "truthfulqa_sk_mc1", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc1_choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + }, + "truthfulqa_sk_mc2": { + "task": "truthfulqa_sk_mc2", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc2_choices", + "process_results": "def process_results_mc2(doc, results):\n lls, is_greedy = zip(*results)\n\n # Split on the first `0` as everything before it is true (`1`).\n split_idx = list(doc[\"mc2_targets\"][\"labels\"]).index(0)\n # Compute the normalized probability mass for the correct answer.\n ll_true, ll_false = lls[:split_idx], lls[split_idx:]\n p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false))\n p_true = p_true / (sum(p_true) + sum(p_false))\n\n return {\"acc\": sum(p_true)}\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + } + }, + "versions": { + "arc_sk": 2.0, + "hellaswag_sk": 1.0, + "m_mmlu_sk": 0.0, + "sklegal": 1, + "skquad": 3, + "truthfulqa_sk_mc1": 1.0, + "truthfulqa_sk_mc2": 1.0 + }, + "n-shot": { + "arc_sk": 0, + "hellaswag_sk": 0, + "m_mmlu_sk": 0, + "sklegal": 0, + "skquad": 0, + "truthfulqa_sk_mc1": 0, + "truthfulqa_sk_mc2": 0 + }, + "higher_is_better": { + "arc_sk": { + "acc": true, + "acc_norm": true + }, + "hellaswag_sk": { + "acc": true, + "acc_norm": true + }, + "m_mmlu_sk": { + "acc": true + }, + "sklegal": { + "acc": true, + "acc_norm": true + }, + "skquad": { + "exact": true, + "f1": true, + "HasAns_exact": true, + "HasAns_f1": true, + "NoAns_exact": true, + "NoAns_f1": true, + "best_exact": true, + "best_f1": true + }, + "truthfulqa_sk_mc1": { + "acc": true + }, + "truthfulqa_sk_mc2": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_sk_mc2": { + "original": 778, + "effective": 778 + }, + "truthfulqa_sk_mc1": { + "original": 778, + "effective": 778 + }, + "skquad": { + "original": 9583, + "effective": 9583 + }, + "sklegal": { + "original": 1334, + "effective": 1334 + }, + "m_mmlu_sk": { + "original": 13062, + "effective": 13062 + }, + "hellaswag_sk": { + "original": 9485, + "effective": 9485 + }, + "arc_sk": { + "original": 1169, + "effective": 1169 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_02_optimizer_adamw8bit_r16_a16_lr5e5,dtype=bfloat16", + "model_num_parameters": 7283675136, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "089497ae0d72018e9591895c54f774bf0b4a83a4", + "peft_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": "6d2abda4", + "date": 1790803446.374072, + "pretty_env_info": "PyTorch version: 2.6.0+cu124\nIs debug build: False\nCUDA used to build PyTorch: 12.4\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.11.16 (main, Aug 27 2026, 14:44:21) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.8.0-138-generic-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA RTX A4500\nGPU 1: NVIDIA RTX A4500\n\nNvidia driver version: 550.163.01\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 16\nOn-line CPU(s) list: 0-15\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) W-2245 CPU @ 3.90GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 8\nSocket(s): 1\nStepping: 7\nCPU(s) scaling MHz: 29%\nCPU max MHz: 4700.0000\nCPU min MHz: 1200.0000\nBogoMIPS: 7799.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 intel_ppin ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm mpx rdt_a avx512f avx512dq rdseed adx smap clflushopt clwb intel_pt avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512_vnni md_clear flush_l1d arch_capabilities ibpb_exit_to_user\nVirtualization: VT-x\nL1d cache: 256 KiB (8 instances)\nL1i cache: 256 KiB (8 instances)\nL2 cache: 8 MiB (8 instances)\nL3 cache: 16.5 MiB (1 instance)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-15\nVulnerability Gather data sampling: Mitigation; Microcode\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX disabled\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Mitigation; Clear CPU buffers; SMT vulnerable\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced / Automatic IBRS; IBPB conditional; PBRSB-eIBRS SW sequence; BHI SW loop, KVM SW loop\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Mitigation; TSX disabled\nVulnerability Vmscape: Mitigation; IBPB before exit to userspace\n\nVersions of relevant libraries:\n[pip3] numpy==2.4.6\n[pip3] nvidia-cublas-cu12==12.4.5.8\n[pip3] nvidia-cuda-cupti-cu12==12.4.127\n[pip3] nvidia-cuda-nvrtc-cu12==12.4.127\n[pip3] nvidia-cuda-runtime-cu12==12.4.127\n[pip3] nvidia-cudnn-cu12==9.1.0.70\n[pip3] nvidia-cufft-cu12==11.2.1.3\n[pip3] nvidia-curand-cu12==10.3.5.147\n[pip3] nvidia-cusolver-cu12==11.6.1.9\n[pip3] nvidia-cusparse-cu12==12.3.1.170\n[pip3] nvidia-cusparselt-cu12==0.6.2\n[pip3] nvidia-nccl-cu12==2.21.5\n[pip3] nvidia-nvjitlink-cu12==12.4.127\n[pip3] nvidia-nvtx-cu12==12.4.127\n[pip3] torch==2.6.0+cu124\n[pip3] triton==3.2.0\n[conda] numpy 2.4.6 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.4.5.8 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.1.0.70 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.2.1.3 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.5.147 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.6.1.9 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.3.1.170 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.6.2 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.21.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.4.127 pypi_0 pypi\n[conda] torch 2.6.0+cu124 pypi_0 pypi\n[conda] triton 3.2.0 pypi_0 pypi", + "transformers_version": "4.48.2", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "", + "2" + ], + "tokenizer_eos_token": [ + "", + "2" + ], + "tokenizer_bos_token": [ + "", + "1" + ], + "eot_token_id": 2, + "max_length": 32768, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/home/schwarc/diplomovka/hruska_compare9_100k_fixed_02_optimizer_adamw8bit_r16_a16_lr5e5", + "model_name_sanitized": "__home__schwarc__diplomovka__hruska_compare9_100k_fixed_02_optimizer_adamw8bit_r16_a16_lr5e5", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 2640480.470625936, + "end_time": 2665645.591866409, + "total_evaluation_time_seconds": "25165.121240472887" +} \ No newline at end of file diff --git a/lmeval/mistral_aya(hruska)/03_rank8_a16_lr5e5/results_2026-10-01T10-25-51.531765.json b/lmeval/mistral_aya(hruska)/03_rank8_a16_lr5e5/results_2026-10-01T10-25-51.531765.json new file mode 100644 index 0000000..61617d2 --- /dev/null +++ b/lmeval/mistral_aya(hruska)/03_rank8_a16_lr5e5/results_2026-10-01T10-25-51.531765.json @@ -0,0 +1,432 @@ +{ + "results": { + "arc_sk": { + "alias": "arc_sk", + "acc,none": 0.28999144568006846, + "acc_stderr,none": 0.013277091943380965, + "acc_norm,none": 0.33105218135158254, + "acc_norm_stderr,none": 0.013769650795341083 + }, + "hellaswag_sk": { + "alias": "hellaswag_sk", + "acc,none": 0.4137058513442277, + "acc_stderr,none": 0.005057173330932102, + "acc_norm,none": 0.5167105956773853, + "acc_norm_stderr,none": 0.005131348933782459 + }, + "m_mmlu_sk": { + "alias": "m_mmlu_sk", + "acc,none": 0.34550604807839536, + "acc_stderr,none": 0.004160949367830717 + }, + "sklegal": { + "alias": "sklegal", + "acc,none": 0.29760119940029983, + "acc_stderr,none": 0.012522589506521369, + "acc_norm,none": 0.4715142428785607, + "acc_norm_stderr,none": 0.013672532895182898 + }, + "skquad": { + "alias": "skquad", + "exact,none": 13.993530209746426, + "exact_stderr,none": "N/A", + "f1,none": 36.36378811056779, + "f1_stderr,none": "N/A", + "HasAns_exact,none": 17.194512116938068, + "HasAns_exact_stderr,none": "N/A", + "HasAns_f1,none": 44.68190556014504, + "HasAns_f1_stderr,none": "N/A", + "NoAns_exact,none": 0.0, + "NoAns_exact_stderr,none": "N/A", + "NoAns_f1,none": 0.0, + "NoAns_f1_stderr,none": "N/A", + "best_exact,none": 18.77282688093499, + "best_exact_stderr,none": "N/A", + "best_f1,none": 36.37507422583225, + "best_f1_stderr,none": "N/A" + }, + "truthfulqa_sk_mc1": { + "alias": "truthfulqa_sk_mc1", + "acc,none": 0.18380462724935734, + "acc_stderr,none": 0.01389519913069634 + }, + "truthfulqa_sk_mc2": { + "alias": "truthfulqa_sk_mc2", + "acc,none": 0.3424520772406517, + "acc_stderr,none": 0.014386406653284318 + } + }, + "group_subtasks": { + "arc_sk": [], + "hellaswag_sk": [], + "m_mmlu_sk": [], + "sklegal": [], + "skquad": [], + "truthfulqa_sk_mc1": [], + "truthfulqa_sk_mc2": [] + }, + "configs": { + "arc_sk": { + "task": "arc_sk", + "tag": [ + "arc_multilingual" + ], + "dataset_path": "alexandrainst/m_arc", + "dataset_name": "sk", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n # breakpoint()\n out_doc = {\n \"id\": doc[\"id\"],\n \"query\": \"Question: \" + preprocess(doc[\"instruction\"]) + \"\\nAnswer:\",\n \"choices\": [\n preprocess(option)\n for option in [\n doc[\"option_a\"],\n doc[\"option_b\"],\n doc[\"option_c\"],\n doc[\"option_d\"],\n doc[\"option_e\"],\n ]\n if option\n ],\n \"gold\": [\"A\", \"B\", \"C\", \"D\", \"E\"].index(doc[\"answer\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "gold", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "query", + "metadata": { + "version": 2.0 + } + }, + "hellaswag_sk": { + "task": "hellaswag_sk", + "tag": [ + "hellaswag_multilingual" + ], + "dataset_path": "alexandrainst/m_hellaswag", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "{{label.lstrip()}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0 + } + }, + "m_mmlu_sk": { + "task": "m_mmlu_sk", + "tag": [ + "m_mmlu" + ], + "dataset_path": "alexandrainst/m_mmlu", + "dataset_name": "sk", + "test_split": "test", + "fewshot_split": "train", + "doc_to_text": "{{instruction.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:", + "doc_to_target": "answer", + "unsafe_code": false, + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n" + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "sklegal": { + "task": "sklegal", + "dataset_path": "TUKE-KEMT/slovak-legal-exam", + "test_split": "test", + "doc_to_text": "{{prompt}} ", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{answers}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1 + } + }, + "skquad": { + "task": "skquad", + "unsafe_code": false, + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "output_type": "generate_until", + "generation_kwargs": { + "until": [ + "\n\n" + ], + "do_sample": false + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 3 + } + }, + "truthfulqa_sk_mc1": { + "task": "truthfulqa_sk_mc1", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc1_choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + }, + "truthfulqa_sk_mc2": { + "task": "truthfulqa_sk_mc2", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc2_choices", + "process_results": "def process_results_mc2(doc, results):\n lls, is_greedy = zip(*results)\n\n # Split on the first `0` as everything before it is true (`1`).\n split_idx = list(doc[\"mc2_targets\"][\"labels\"]).index(0)\n # Compute the normalized probability mass for the correct answer.\n ll_true, ll_false = lls[:split_idx], lls[split_idx:]\n p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false))\n p_true = p_true / (sum(p_true) + sum(p_false))\n\n return {\"acc\": sum(p_true)}\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + } + }, + "versions": { + "arc_sk": 2.0, + "hellaswag_sk": 1.0, + "m_mmlu_sk": 0.0, + "sklegal": 1, + "skquad": 3, + "truthfulqa_sk_mc1": 1.0, + "truthfulqa_sk_mc2": 1.0 + }, + "n-shot": { + "arc_sk": 0, + "hellaswag_sk": 0, + "m_mmlu_sk": 0, + "sklegal": 0, + "skquad": 0, + "truthfulqa_sk_mc1": 0, + "truthfulqa_sk_mc2": 0 + }, + "higher_is_better": { + "arc_sk": { + "acc": true, + "acc_norm": true + }, + "hellaswag_sk": { + "acc": true, + "acc_norm": true + }, + "m_mmlu_sk": { + "acc": true + }, + "sklegal": { + "acc": true, + "acc_norm": true + }, + "skquad": { + "exact": true, + "f1": true, + "HasAns_exact": true, + "HasAns_f1": true, + "NoAns_exact": true, + "NoAns_f1": true, + "best_exact": true, + "best_f1": true + }, + "truthfulqa_sk_mc1": { + "acc": true + }, + "truthfulqa_sk_mc2": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_sk_mc2": { + "original": 778, + "effective": 778 + }, + "truthfulqa_sk_mc1": { + "original": 778, + "effective": 778 + }, + "skquad": { + "original": 9583, + "effective": 9583 + }, + "sklegal": { + "original": 1334, + "effective": 1334 + }, + "m_mmlu_sk": { + "original": 13062, + "effective": 13062 + }, + "hellaswag_sk": { + "original": 9485, + "effective": 9485 + }, + "arc_sk": { + "original": 1169, + "effective": 1169 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_03_rank8_a16_lr5e5,dtype=bfloat16", + "model_num_parameters": 7262703616, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "089497ae0d72018e9591895c54f774bf0b4a83a4", + "peft_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": "6d2abda4", + "date": 1790828636.6982293, + "pretty_env_info": "PyTorch version: 2.6.0+cu124\nIs debug build: False\nCUDA used to build PyTorch: 12.4\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.11.16 (main, Aug 27 2026, 14:44:21) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.8.0-138-generic-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA RTX A4500\nGPU 1: NVIDIA RTX A4500\n\nNvidia driver version: 550.163.01\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 16\nOn-line CPU(s) list: 0-15\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) W-2245 CPU @ 3.90GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 8\nSocket(s): 1\nStepping: 7\nCPU(s) scaling MHz: 28%\nCPU max MHz: 4700.0000\nCPU min MHz: 1200.0000\nBogoMIPS: 7799.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 intel_ppin ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm mpx rdt_a avx512f avx512dq rdseed adx smap clflushopt clwb intel_pt avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512_vnni md_clear flush_l1d arch_capabilities ibpb_exit_to_user\nVirtualization: VT-x\nL1d cache: 256 KiB (8 instances)\nL1i cache: 256 KiB (8 instances)\nL2 cache: 8 MiB (8 instances)\nL3 cache: 16.5 MiB (1 instance)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-15\nVulnerability Gather data sampling: Mitigation; Microcode\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX disabled\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Mitigation; Clear CPU buffers; SMT vulnerable\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced / Automatic IBRS; IBPB conditional; PBRSB-eIBRS SW sequence; BHI SW loop, KVM SW loop\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Mitigation; TSX disabled\nVulnerability Vmscape: Mitigation; IBPB before exit to userspace\n\nVersions of relevant libraries:\n[pip3] numpy==2.4.6\n[pip3] nvidia-cublas-cu12==12.4.5.8\n[pip3] nvidia-cuda-cupti-cu12==12.4.127\n[pip3] nvidia-cuda-nvrtc-cu12==12.4.127\n[pip3] nvidia-cuda-runtime-cu12==12.4.127\n[pip3] nvidia-cudnn-cu12==9.1.0.70\n[pip3] nvidia-cufft-cu12==11.2.1.3\n[pip3] nvidia-curand-cu12==10.3.5.147\n[pip3] nvidia-cusolver-cu12==11.6.1.9\n[pip3] nvidia-cusparse-cu12==12.3.1.170\n[pip3] nvidia-cusparselt-cu12==0.6.2\n[pip3] nvidia-nccl-cu12==2.21.5\n[pip3] nvidia-nvjitlink-cu12==12.4.127\n[pip3] nvidia-nvtx-cu12==12.4.127\n[pip3] torch==2.6.0+cu124\n[pip3] triton==3.2.0\n[conda] numpy 2.4.6 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.4.5.8 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.1.0.70 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.2.1.3 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.5.147 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.6.1.9 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.3.1.170 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.6.2 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.21.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.4.127 pypi_0 pypi\n[conda] torch 2.6.0+cu124 pypi_0 pypi\n[conda] triton 3.2.0 pypi_0 pypi", + "transformers_version": "4.48.2", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "", + "2" + ], + "tokenizer_eos_token": [ + "", + "2" + ], + "tokenizer_bos_token": [ + "", + "1" + ], + "eot_token_id": 2, + "max_length": 32768, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/home/schwarc/diplomovka/hruska_compare9_100k_fixed_03_rank8_a16_lr5e5", + "model_name_sanitized": "__home__schwarc__diplomovka__hruska_compare9_100k_fixed_03_rank8_a16_lr5e5", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 2665670.889294327, + "end_time": 2687393.332034511, + "total_evaluation_time_seconds": "21722.442740184255" +} \ No newline at end of file diff --git a/lmeval/mistral_aya(hruska)/04_rank32_a32_lr5e5/results_2026-10-01T17-22-48.146390.json b/lmeval/mistral_aya(hruska)/04_rank32_a32_lr5e5/results_2026-10-01T17-22-48.146390.json new file mode 100644 index 0000000..e3f7781 --- /dev/null +++ b/lmeval/mistral_aya(hruska)/04_rank32_a32_lr5e5/results_2026-10-01T17-22-48.146390.json @@ -0,0 +1,432 @@ +{ + "results": { + "arc_sk": { + "alias": "arc_sk", + "acc,none": 0.2891360136869119, + "acc_stderr,none": 0.013265478787531884, + "acc_norm,none": 0.33532934131736525, + "acc_norm_stderr,none": 0.013813941431917236 + }, + "hellaswag_sk": { + "alias": "hellaswag_sk", + "acc,none": 0.41412756984712706, + "acc_stderr,none": 0.005057930180607637, + "acc_norm,none": 0.5200843437005799, + "acc_norm_stderr,none": 0.005130073371619102 + }, + "m_mmlu_sk": { + "alias": "m_mmlu_sk", + "acc,none": 0.3449701423977951, + "acc_stderr,none": 0.004159422985849433 + }, + "sklegal": { + "alias": "sklegal", + "acc,none": 0.2908545727136432, + "acc_stderr,none": 0.012439144896870835, + "acc_norm,none": 0.47301349325337333, + "acc_norm_stderr,none": 0.013674814312334499 + }, + "skquad": { + "alias": "skquad", + "exact,none": 13.32568089324846, + "exact_stderr,none": "N/A", + "f1,none": 36.02282635500946, + "f1_stderr,none": "N/A", + "HasAns_exact,none": 16.361071932299012, + "HasAns_exact_stderr,none": "N/A", + "HasAns_f1,none": 44.2501275753373, + "HasAns_f1_stderr,none": "N/A", + "NoAns_exact,none": 0.05605381165919283, + "NoAns_exact_stderr,none": "N/A", + "NoAns_f1,none": 0.05605381165919283, + "NoAns_f1_stderr,none": "N/A", + "best_exact,none": 18.67891057080246, + "best_exact_stderr,none": "N/A", + "best_f1,none": 36.12444280653584, + "best_f1_stderr,none": "N/A" + }, + "truthfulqa_sk_mc1": { + "alias": "truthfulqa_sk_mc1", + "acc,none": 0.19023136246786632, + "acc_stderr,none": 0.014080271401161148 + }, + "truthfulqa_sk_mc2": { + "alias": "truthfulqa_sk_mc2", + "acc,none": 0.3367013908863517, + "acc_stderr,none": 0.014250354342251092 + } + }, + "group_subtasks": { + "arc_sk": [], + "hellaswag_sk": [], + "m_mmlu_sk": [], + "sklegal": [], + "skquad": [], + "truthfulqa_sk_mc1": [], + "truthfulqa_sk_mc2": [] + }, + "configs": { + "arc_sk": { + "task": "arc_sk", + "tag": [ + "arc_multilingual" + ], + "dataset_path": "alexandrainst/m_arc", + "dataset_name": "sk", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n # breakpoint()\n out_doc = {\n \"id\": doc[\"id\"],\n \"query\": \"Question: \" + preprocess(doc[\"instruction\"]) + \"\\nAnswer:\",\n \"choices\": [\n preprocess(option)\n for option in [\n doc[\"option_a\"],\n doc[\"option_b\"],\n doc[\"option_c\"],\n doc[\"option_d\"],\n doc[\"option_e\"],\n ]\n if option\n ],\n \"gold\": [\"A\", \"B\", \"C\", \"D\", \"E\"].index(doc[\"answer\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "gold", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "query", + "metadata": { + "version": 2.0 + } + }, + "hellaswag_sk": { + "task": "hellaswag_sk", + "tag": [ + "hellaswag_multilingual" + ], + "dataset_path": "alexandrainst/m_hellaswag", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "{{label.lstrip()}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0 + } + }, + "m_mmlu_sk": { + "task": "m_mmlu_sk", + "tag": [ + "m_mmlu" + ], + "dataset_path": "alexandrainst/m_mmlu", + "dataset_name": "sk", + "test_split": "test", + "fewshot_split": "train", + "doc_to_text": "{{instruction.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:", + "doc_to_target": "answer", + "unsafe_code": false, + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n" + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "sklegal": { + "task": "sklegal", + "dataset_path": "TUKE-KEMT/slovak-legal-exam", + "test_split": "test", + "doc_to_text": "{{prompt}} ", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{answers}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1 + } + }, + "skquad": { + "task": "skquad", + "unsafe_code": false, + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "output_type": "generate_until", + "generation_kwargs": { + "until": [ + "\n\n" + ], + "do_sample": false + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 3 + } + }, + "truthfulqa_sk_mc1": { + "task": "truthfulqa_sk_mc1", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc1_choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + }, + "truthfulqa_sk_mc2": { + "task": "truthfulqa_sk_mc2", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc2_choices", + "process_results": "def process_results_mc2(doc, results):\n lls, is_greedy = zip(*results)\n\n # Split on the first `0` as everything before it is true (`1`).\n split_idx = list(doc[\"mc2_targets\"][\"labels\"]).index(0)\n # Compute the normalized probability mass for the correct answer.\n ll_true, ll_false = lls[:split_idx], lls[split_idx:]\n p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false))\n p_true = p_true / (sum(p_true) + sum(p_false))\n\n return {\"acc\": sum(p_true)}\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + } + }, + "versions": { + "arc_sk": 2.0, + "hellaswag_sk": 1.0, + "m_mmlu_sk": 0.0, + "sklegal": 1, + "skquad": 3, + "truthfulqa_sk_mc1": 1.0, + "truthfulqa_sk_mc2": 1.0 + }, + "n-shot": { + "arc_sk": 0, + "hellaswag_sk": 0, + "m_mmlu_sk": 0, + "sklegal": 0, + "skquad": 0, + "truthfulqa_sk_mc1": 0, + "truthfulqa_sk_mc2": 0 + }, + "higher_is_better": { + "arc_sk": { + "acc": true, + "acc_norm": true + }, + "hellaswag_sk": { + "acc": true, + "acc_norm": true + }, + "m_mmlu_sk": { + "acc": true + }, + "sklegal": { + "acc": true, + "acc_norm": true + }, + "skquad": { + "exact": true, + "f1": true, + "HasAns_exact": true, + "HasAns_f1": true, + "NoAns_exact": true, + "NoAns_f1": true, + "best_exact": true, + "best_f1": true + }, + "truthfulqa_sk_mc1": { + "acc": true + }, + "truthfulqa_sk_mc2": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_sk_mc2": { + "original": 778, + "effective": 778 + }, + "truthfulqa_sk_mc1": { + "original": 778, + "effective": 778 + }, + "skquad": { + "original": 9583, + "effective": 9583 + }, + "sklegal": { + "original": 1334, + "effective": 1334 + }, + "m_mmlu_sk": { + "original": 13062, + "effective": 13062 + }, + "hellaswag_sk": { + "original": 9485, + "effective": 9485 + }, + "arc_sk": { + "original": 1169, + "effective": 1169 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_04_rank32_a32_lr5e5,dtype=bfloat16", + "model_num_parameters": 7325618176, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "089497ae0d72018e9591895c54f774bf0b4a83a4", + "peft_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": "6d2abda4", + "date": 1790850384.5766006, + "pretty_env_info": "PyTorch version: 2.6.0+cu124\nIs debug build: False\nCUDA used to build PyTorch: 12.4\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.11.16 (main, Aug 27 2026, 14:44:21) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.8.0-138-generic-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA RTX A4500\nGPU 1: NVIDIA RTX A4500\n\nNvidia driver version: 550.163.01\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 16\nOn-line CPU(s) list: 0-15\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) W-2245 CPU @ 3.90GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 8\nSocket(s): 1\nStepping: 7\nCPU(s) scaling MHz: 29%\nCPU max MHz: 4700.0000\nCPU min MHz: 1200.0000\nBogoMIPS: 7799.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 intel_ppin ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm mpx rdt_a avx512f avx512dq rdseed adx smap clflushopt clwb intel_pt avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512_vnni md_clear flush_l1d arch_capabilities ibpb_exit_to_user\nVirtualization: VT-x\nL1d cache: 256 KiB (8 instances)\nL1i cache: 256 KiB (8 instances)\nL2 cache: 8 MiB (8 instances)\nL3 cache: 16.5 MiB (1 instance)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-15\nVulnerability Gather data sampling: Mitigation; Microcode\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX disabled\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Mitigation; Clear CPU buffers; SMT vulnerable\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced / Automatic IBRS; IBPB conditional; PBRSB-eIBRS SW sequence; BHI SW loop, KVM SW loop\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Mitigation; TSX disabled\nVulnerability Vmscape: Mitigation; IBPB before exit to userspace\n\nVersions of relevant libraries:\n[pip3] numpy==2.4.6\n[pip3] nvidia-cublas-cu12==12.4.5.8\n[pip3] nvidia-cuda-cupti-cu12==12.4.127\n[pip3] nvidia-cuda-nvrtc-cu12==12.4.127\n[pip3] nvidia-cuda-runtime-cu12==12.4.127\n[pip3] nvidia-cudnn-cu12==9.1.0.70\n[pip3] nvidia-cufft-cu12==11.2.1.3\n[pip3] nvidia-curand-cu12==10.3.5.147\n[pip3] nvidia-cusolver-cu12==11.6.1.9\n[pip3] nvidia-cusparse-cu12==12.3.1.170\n[pip3] nvidia-cusparselt-cu12==0.6.2\n[pip3] nvidia-nccl-cu12==2.21.5\n[pip3] nvidia-nvjitlink-cu12==12.4.127\n[pip3] nvidia-nvtx-cu12==12.4.127\n[pip3] torch==2.6.0+cu124\n[pip3] triton==3.2.0\n[conda] numpy 2.4.6 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.4.5.8 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.1.0.70 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.2.1.3 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.5.147 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.6.1.9 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.3.1.170 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.6.2 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.21.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.4.127 pypi_0 pypi\n[conda] torch 2.6.0+cu124 pypi_0 pypi\n[conda] triton 3.2.0 pypi_0 pypi", + "transformers_version": "4.48.2", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "", + "2" + ], + "tokenizer_eos_token": [ + "", + "2" + ], + "tokenizer_bos_token": [ + "", + "1" + ], + "eot_token_id": 2, + "max_length": 32768, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/home/schwarc/diplomovka/hruska_compare9_100k_fixed_04_rank32_a32_lr5e5", + "model_name_sanitized": "__home__schwarc__diplomovka__hruska_compare9_100k_fixed_04_rank32_a32_lr5e5", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 2687418.662174277, + "end_time": 2712409.94649, + "total_evaluation_time_seconds": "24991.284315722995" +} \ No newline at end of file diff --git a/lmeval/mistral_aya(hruska)/05_r16_alpha32_lr5e5/results_2026-10-02T00-19-55.725179.json b/lmeval/mistral_aya(hruska)/05_r16_alpha32_lr5e5/results_2026-10-02T00-19-55.725179.json new file mode 100644 index 0000000..888989d --- /dev/null +++ b/lmeval/mistral_aya(hruska)/05_r16_alpha32_lr5e5/results_2026-10-02T00-19-55.725179.json @@ -0,0 +1,432 @@ +{ + "results": { + "arc_sk": { + "alias": "arc_sk", + "acc,none": 0.2994011976047904, + "acc_stderr,none": 0.013401087427687511, + "acc_norm,none": 0.33361847733105215, + "acc_norm_stderr,none": 0.013796378474151154 + }, + "hellaswag_sk": { + "alias": "hellaswag_sk", + "acc,none": 0.4130732735898788, + "acc_stderr,none": 0.005056030889826752, + "acc_norm,none": 0.5190300474433316, + "acc_norm_stderr,none": 0.0051304971474509355 + }, + "m_mmlu_sk": { + "alias": "m_mmlu_sk", + "acc,none": 0.35714285714285715, + "acc_stderr,none": 0.004192663488668836 + }, + "sklegal": { + "alias": "sklegal", + "acc,none": 0.29685157421289354, + "acc_stderr,none": 0.012513480116311385, + "acc_norm,none": 0.46701649175412296, + "acc_norm_stderr,none": 0.013664945986905805 + }, + "skquad": { + "alias": "skquad", + "exact,none": 14.598768652822708, + "exact_stderr,none": "N/A", + "f1,none": 37.873456685535025, + "f1_stderr,none": "N/A", + "HasAns_exact,none": 17.925375048083087, + "HasAns_exact_stderr,none": "N/A", + "HasAns_f1,none": 46.52408455154278, + "HasAns_f1_stderr,none": "N/A", + "NoAns_exact,none": 0.05605381165919283, + "NoAns_exact_stderr,none": "N/A", + "NoAns_f1,none": 0.05605381165919283, + "NoAns_f1_stderr,none": "N/A", + "best_exact,none": 18.77282688093499, + "best_exact_stderr,none": "N/A", + "best_f1,none": 38.068000672246654, + "best_f1_stderr,none": "N/A" + }, + "truthfulqa_sk_mc1": { + "alias": "truthfulqa_sk_mc1", + "acc,none": 0.19665809768637532, + "acc_stderr,none": 0.01425921501143319 + }, + "truthfulqa_sk_mc2": { + "alias": "truthfulqa_sk_mc2", + "acc,none": 0.3555125811399826, + "acc_stderr,none": 0.014603750406212533 + } + }, + "group_subtasks": { + "arc_sk": [], + "hellaswag_sk": [], + "m_mmlu_sk": [], + "sklegal": [], + "skquad": [], + "truthfulqa_sk_mc1": [], + "truthfulqa_sk_mc2": [] + }, + "configs": { + "arc_sk": { + "task": "arc_sk", + "tag": [ + "arc_multilingual" + ], + "dataset_path": "alexandrainst/m_arc", + "dataset_name": "sk", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n # breakpoint()\n out_doc = {\n \"id\": doc[\"id\"],\n \"query\": \"Question: \" + preprocess(doc[\"instruction\"]) + \"\\nAnswer:\",\n \"choices\": [\n preprocess(option)\n for option in [\n doc[\"option_a\"],\n doc[\"option_b\"],\n doc[\"option_c\"],\n doc[\"option_d\"],\n doc[\"option_e\"],\n ]\n if option\n ],\n \"gold\": [\"A\", \"B\", \"C\", \"D\", \"E\"].index(doc[\"answer\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "gold", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "query", + "metadata": { + "version": 2.0 + } + }, + "hellaswag_sk": { + "task": "hellaswag_sk", + "tag": [ + "hellaswag_multilingual" + ], + "dataset_path": "alexandrainst/m_hellaswag", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "{{label.lstrip()}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0 + } + }, + "m_mmlu_sk": { + "task": "m_mmlu_sk", + "tag": [ + "m_mmlu" + ], + "dataset_path": "alexandrainst/m_mmlu", + "dataset_name": "sk", + "test_split": "test", + "fewshot_split": "train", + "doc_to_text": "{{instruction.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:", + "doc_to_target": "answer", + "unsafe_code": false, + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n" + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "sklegal": { + "task": "sklegal", + "dataset_path": "TUKE-KEMT/slovak-legal-exam", + "test_split": "test", + "doc_to_text": "{{prompt}} ", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{answers}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1 + } + }, + "skquad": { + "task": "skquad", + "unsafe_code": false, + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "output_type": "generate_until", + "generation_kwargs": { + "until": [ + "\n\n" + ], + "do_sample": false + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 3 + } + }, + "truthfulqa_sk_mc1": { + "task": "truthfulqa_sk_mc1", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc1_choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + }, + "truthfulqa_sk_mc2": { + "task": "truthfulqa_sk_mc2", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc2_choices", + "process_results": "def process_results_mc2(doc, results):\n lls, is_greedy = zip(*results)\n\n # Split on the first `0` as everything before it is true (`1`).\n split_idx = list(doc[\"mc2_targets\"][\"labels\"]).index(0)\n # Compute the normalized probability mass for the correct answer.\n ll_true, ll_false = lls[:split_idx], lls[split_idx:]\n p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false))\n p_true = p_true / (sum(p_true) + sum(p_false))\n\n return {\"acc\": sum(p_true)}\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + } + }, + "versions": { + "arc_sk": 2.0, + "hellaswag_sk": 1.0, + "m_mmlu_sk": 0.0, + "sklegal": 1, + "skquad": 3, + "truthfulqa_sk_mc1": 1.0, + "truthfulqa_sk_mc2": 1.0 + }, + "n-shot": { + "arc_sk": 0, + "hellaswag_sk": 0, + "m_mmlu_sk": 0, + "sklegal": 0, + "skquad": 0, + "truthfulqa_sk_mc1": 0, + "truthfulqa_sk_mc2": 0 + }, + "higher_is_better": { + "arc_sk": { + "acc": true, + "acc_norm": true + }, + "hellaswag_sk": { + "acc": true, + "acc_norm": true + }, + "m_mmlu_sk": { + "acc": true + }, + "sklegal": { + "acc": true, + "acc_norm": true + }, + "skquad": { + "exact": true, + "f1": true, + "HasAns_exact": true, + "HasAns_f1": true, + "NoAns_exact": true, + "NoAns_f1": true, + "best_exact": true, + "best_f1": true + }, + "truthfulqa_sk_mc1": { + "acc": true + }, + "truthfulqa_sk_mc2": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_sk_mc2": { + "original": 778, + "effective": 778 + }, + "truthfulqa_sk_mc1": { + "original": 778, + "effective": 778 + }, + "skquad": { + "original": 9583, + "effective": 9583 + }, + "sklegal": { + "original": 1334, + "effective": 1334 + }, + "m_mmlu_sk": { + "original": 13062, + "effective": 13062 + }, + "hellaswag_sk": { + "original": 9485, + "effective": 9485 + }, + "arc_sk": { + "original": 1169, + "effective": 1169 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_05_r16_alpha32_lr5e5,dtype=bfloat16", + "model_num_parameters": 7283675136, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "089497ae0d72018e9591895c54f774bf0b4a83a4", + "peft_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": "6d2abda4", + "date": 1790875401.1466277, + "pretty_env_info": "PyTorch version: 2.6.0+cu124\nIs debug build: False\nCUDA used to build PyTorch: 12.4\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.11.16 (main, Aug 27 2026, 14:44:21) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.8.0-138-generic-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA RTX A4500\nGPU 1: NVIDIA RTX A4500\n\nNvidia driver version: 550.163.01\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 16\nOn-line CPU(s) list: 0-15\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) W-2245 CPU @ 3.90GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 8\nSocket(s): 1\nStepping: 7\nCPU(s) scaling MHz: 27%\nCPU max MHz: 4700.0000\nCPU min MHz: 1200.0000\nBogoMIPS: 7799.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 intel_ppin ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm mpx rdt_a avx512f avx512dq rdseed adx smap clflushopt clwb intel_pt avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512_vnni md_clear flush_l1d arch_capabilities ibpb_exit_to_user\nVirtualization: VT-x\nL1d cache: 256 KiB (8 instances)\nL1i cache: 256 KiB (8 instances)\nL2 cache: 8 MiB (8 instances)\nL3 cache: 16.5 MiB (1 instance)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-15\nVulnerability Gather data sampling: Mitigation; Microcode\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX disabled\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Mitigation; Clear CPU buffers; SMT vulnerable\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced / Automatic IBRS; IBPB conditional; PBRSB-eIBRS SW sequence; BHI SW loop, KVM SW loop\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Mitigation; TSX disabled\nVulnerability Vmscape: Mitigation; IBPB before exit to userspace\n\nVersions of relevant libraries:\n[pip3] numpy==2.4.6\n[pip3] nvidia-cublas-cu12==12.4.5.8\n[pip3] nvidia-cuda-cupti-cu12==12.4.127\n[pip3] nvidia-cuda-nvrtc-cu12==12.4.127\n[pip3] nvidia-cuda-runtime-cu12==12.4.127\n[pip3] nvidia-cudnn-cu12==9.1.0.70\n[pip3] nvidia-cufft-cu12==11.2.1.3\n[pip3] nvidia-curand-cu12==10.3.5.147\n[pip3] nvidia-cusolver-cu12==11.6.1.9\n[pip3] nvidia-cusparse-cu12==12.3.1.170\n[pip3] nvidia-cusparselt-cu12==0.6.2\n[pip3] nvidia-nccl-cu12==2.21.5\n[pip3] nvidia-nvjitlink-cu12==12.4.127\n[pip3] nvidia-nvtx-cu12==12.4.127\n[pip3] torch==2.6.0+cu124\n[pip3] triton==3.2.0\n[conda] numpy 2.4.6 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.4.5.8 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.1.0.70 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.2.1.3 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.5.147 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.6.1.9 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.3.1.170 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.6.2 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.21.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.4.127 pypi_0 pypi\n[conda] torch 2.6.0+cu124 pypi_0 pypi\n[conda] triton 3.2.0 pypi_0 pypi", + "transformers_version": "4.48.2", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "", + "2" + ], + "tokenizer_eos_token": [ + "", + "2" + ], + "tokenizer_bos_token": [ + "", + "1" + ], + "eot_token_id": 2, + "max_length": 32768, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/home/schwarc/diplomovka/hruska_compare9_100k_fixed_05_r16_alpha32_lr5e5", + "model_name_sanitized": "__home__schwarc__diplomovka__hruska_compare9_100k_fixed_05_r16_alpha32_lr5e5", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 2712435.21004293, + "end_time": 2737437.52530566, + "total_evaluation_time_seconds": "25002.315262729768" +} \ No newline at end of file diff --git a/lmeval/mistral_aya(hruska)/06_lr1e4_r16_a16/results_2026-10-02T06-59-24.237777.json b/lmeval/mistral_aya(hruska)/06_lr1e4_r16_a16/results_2026-10-02T06-59-24.237777.json new file mode 100644 index 0000000..26a16ad --- /dev/null +++ b/lmeval/mistral_aya(hruska)/06_lr1e4_r16_a16/results_2026-10-02T06-59-24.237777.json @@ -0,0 +1,432 @@ +{ + "results": { + "arc_sk": { + "alias": "arc_sk", + "acc,none": 0.29597946963216426, + "acc_stderr,none": 0.013356788048643691, + "acc_norm,none": 0.33105218135158254, + "acc_norm_stderr,none": 0.013769650795341088 + }, + "hellaswag_sk": { + "alias": "hellaswag_sk", + "acc,none": 0.411070110701107, + "acc_stderr,none": 0.005052356361834433, + "acc_norm,none": 0.5207169214549289, + "acc_norm_stderr,none": 0.005129808122680316 + }, + "m_mmlu_sk": { + "alias": "m_mmlu_sk", + "acc,none": 0.3549992344204563, + "acc_stderr,none": 0.004187025526122449 + }, + "sklegal": { + "alias": "sklegal", + "acc,none": 0.2841079460269865, + "acc_stderr,none": 0.01235237268913334, + "acc_norm,none": 0.46551724137931033, + "acc_norm_stderr,none": 0.013662169236087337 + }, + "skquad": { + "alias": "skquad", + "exact,none": 14.014400500886987, + "exact_stderr,none": "N/A", + "f1,none": 37.57962398015711, + "f1_stderr,none": "N/A", + "HasAns_exact,none": 17.207334273624824, + "HasAns_exact_stderr,none": "N/A", + "HasAns_f1,none": 46.16303841541808, + "HasAns_f1_stderr,none": "N/A", + "NoAns_exact,none": 0.05605381165919283, + "NoAns_exact_stderr,none": "N/A", + "NoAns_f1,none": 0.05605381165919283, + "NoAns_f1_stderr,none": "N/A", + "best_exact,none": 18.80413231764583, + "best_exact_stderr,none": "N/A", + "best_f1,none": 37.69906094942537, + "best_f1_stderr,none": "N/A" + }, + "truthfulqa_sk_mc1": { + "alias": "truthfulqa_sk_mc1", + "acc,none": 0.19023136246786632, + "acc_stderr,none": 0.014080271401161151 + }, + "truthfulqa_sk_mc2": { + "alias": "truthfulqa_sk_mc2", + "acc,none": 0.34733281230911506, + "acc_stderr,none": 0.014356264797779777 + } + }, + "group_subtasks": { + "arc_sk": [], + "hellaswag_sk": [], + "m_mmlu_sk": [], + "sklegal": [], + "skquad": [], + "truthfulqa_sk_mc1": [], + "truthfulqa_sk_mc2": [] + }, + "configs": { + "arc_sk": { + "task": "arc_sk", + "tag": [ + "arc_multilingual" + ], + "dataset_path": "alexandrainst/m_arc", + "dataset_name": "sk", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n # breakpoint()\n out_doc = {\n \"id\": doc[\"id\"],\n \"query\": \"Question: \" + preprocess(doc[\"instruction\"]) + \"\\nAnswer:\",\n \"choices\": [\n preprocess(option)\n for option in [\n doc[\"option_a\"],\n doc[\"option_b\"],\n doc[\"option_c\"],\n doc[\"option_d\"],\n doc[\"option_e\"],\n ]\n if option\n ],\n \"gold\": [\"A\", \"B\", \"C\", \"D\", \"E\"].index(doc[\"answer\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "gold", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "query", + "metadata": { + "version": 2.0 + } + }, + "hellaswag_sk": { + "task": "hellaswag_sk", + "tag": [ + "hellaswag_multilingual" + ], + "dataset_path": "alexandrainst/m_hellaswag", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "{{label.lstrip()}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0 + } + }, + "m_mmlu_sk": { + "task": "m_mmlu_sk", + "tag": [ + "m_mmlu" + ], + "dataset_path": "alexandrainst/m_mmlu", + "dataset_name": "sk", + "test_split": "test", + "fewshot_split": "train", + "doc_to_text": "{{instruction.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:", + "doc_to_target": "answer", + "unsafe_code": false, + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n" + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "sklegal": { + "task": "sklegal", + "dataset_path": "TUKE-KEMT/slovak-legal-exam", + "test_split": "test", + "doc_to_text": "{{prompt}} ", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{answers}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1 + } + }, + "skquad": { + "task": "skquad", + "unsafe_code": false, + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "output_type": "generate_until", + "generation_kwargs": { + "until": [ + "\n\n" + ], + "do_sample": false + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 3 + } + }, + "truthfulqa_sk_mc1": { + "task": "truthfulqa_sk_mc1", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc1_choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + }, + "truthfulqa_sk_mc2": { + "task": "truthfulqa_sk_mc2", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc2_choices", + "process_results": "def process_results_mc2(doc, results):\n lls, is_greedy = zip(*results)\n\n # Split on the first `0` as everything before it is true (`1`).\n split_idx = list(doc[\"mc2_targets\"][\"labels\"]).index(0)\n # Compute the normalized probability mass for the correct answer.\n ll_true, ll_false = lls[:split_idx], lls[split_idx:]\n p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false))\n p_true = p_true / (sum(p_true) + sum(p_false))\n\n return {\"acc\": sum(p_true)}\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + } + }, + "versions": { + "arc_sk": 2.0, + "hellaswag_sk": 1.0, + "m_mmlu_sk": 0.0, + "sklegal": 1, + "skquad": 3, + "truthfulqa_sk_mc1": 1.0, + "truthfulqa_sk_mc2": 1.0 + }, + "n-shot": { + "arc_sk": 0, + "hellaswag_sk": 0, + "m_mmlu_sk": 0, + "sklegal": 0, + "skquad": 0, + "truthfulqa_sk_mc1": 0, + "truthfulqa_sk_mc2": 0 + }, + "higher_is_better": { + "arc_sk": { + "acc": true, + "acc_norm": true + }, + "hellaswag_sk": { + "acc": true, + "acc_norm": true + }, + "m_mmlu_sk": { + "acc": true + }, + "sklegal": { + "acc": true, + "acc_norm": true + }, + "skquad": { + "exact": true, + "f1": true, + "HasAns_exact": true, + "HasAns_f1": true, + "NoAns_exact": true, + "NoAns_f1": true, + "best_exact": true, + "best_f1": true + }, + "truthfulqa_sk_mc1": { + "acc": true + }, + "truthfulqa_sk_mc2": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_sk_mc2": { + "original": 778, + "effective": 778 + }, + "truthfulqa_sk_mc1": { + "original": 778, + "effective": 778 + }, + "skquad": { + "original": 9583, + "effective": 9583 + }, + "sklegal": { + "original": 1334, + "effective": 1334 + }, + "m_mmlu_sk": { + "original": 13062, + "effective": 13062 + }, + "hellaswag_sk": { + "original": 9485, + "effective": 9485 + }, + "arc_sk": { + "original": 1169, + "effective": 1169 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_06_lr1e4_r16_a16,dtype=bfloat16", + "model_num_parameters": 7283675136, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "089497ae0d72018e9591895c54f774bf0b4a83a4", + "peft_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": "6d2abda4", + "date": 1790900428.7180088, + "pretty_env_info": "PyTorch version: 2.6.0+cu124\nIs debug build: False\nCUDA used to build PyTorch: 12.4\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.11.16 (main, Aug 27 2026, 14:44:21) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.8.0-138-generic-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA RTX A4500\nGPU 1: NVIDIA RTX A4500\n\nNvidia driver version: 550.163.01\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 16\nOn-line CPU(s) list: 0-15\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) W-2245 CPU @ 3.90GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 8\nSocket(s): 1\nStepping: 7\nCPU(s) scaling MHz: 26%\nCPU max MHz: 4700.0000\nCPU min MHz: 1200.0000\nBogoMIPS: 7799.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 intel_ppin ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm mpx rdt_a avx512f avx512dq rdseed adx smap clflushopt clwb intel_pt avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512_vnni md_clear flush_l1d arch_capabilities ibpb_exit_to_user\nVirtualization: VT-x\nL1d cache: 256 KiB (8 instances)\nL1i cache: 256 KiB (8 instances)\nL2 cache: 8 MiB (8 instances)\nL3 cache: 16.5 MiB (1 instance)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-15\nVulnerability Gather data sampling: Mitigation; Microcode\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX disabled\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Mitigation; Clear CPU buffers; SMT vulnerable\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced / Automatic IBRS; IBPB conditional; PBRSB-eIBRS SW sequence; BHI SW loop, KVM SW loop\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Mitigation; TSX disabled\nVulnerability Vmscape: Mitigation; IBPB before exit to userspace\n\nVersions of relevant libraries:\n[pip3] numpy==2.4.6\n[pip3] nvidia-cublas-cu12==12.4.5.8\n[pip3] nvidia-cuda-cupti-cu12==12.4.127\n[pip3] nvidia-cuda-nvrtc-cu12==12.4.127\n[pip3] nvidia-cuda-runtime-cu12==12.4.127\n[pip3] nvidia-cudnn-cu12==9.1.0.70\n[pip3] nvidia-cufft-cu12==11.2.1.3\n[pip3] nvidia-curand-cu12==10.3.5.147\n[pip3] nvidia-cusolver-cu12==11.6.1.9\n[pip3] nvidia-cusparse-cu12==12.3.1.170\n[pip3] nvidia-cusparselt-cu12==0.6.2\n[pip3] nvidia-nccl-cu12==2.21.5\n[pip3] nvidia-nvjitlink-cu12==12.4.127\n[pip3] nvidia-nvtx-cu12==12.4.127\n[pip3] torch==2.6.0+cu124\n[pip3] triton==3.2.0\n[conda] numpy 2.4.6 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.4.5.8 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.1.0.70 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.2.1.3 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.5.147 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.6.1.9 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.3.1.170 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.6.2 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.21.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.4.127 pypi_0 pypi\n[conda] torch 2.6.0+cu124 pypi_0 pypi\n[conda] triton 3.2.0 pypi_0 pypi", + "transformers_version": "4.48.2", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "", + "2" + ], + "tokenizer_eos_token": [ + "", + "2" + ], + "tokenizer_bos_token": [ + "", + "1" + ], + "eot_token_id": 2, + "max_length": 32768, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/home/schwarc/diplomovka/hruska_compare9_100k_fixed_06_lr1e4_r16_a16", + "model_name_sanitized": "__home__schwarc__diplomovka__hruska_compare9_100k_fixed_06_lr1e4_r16_a16", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 2737462.795686631, + "end_time": 2761406.037857553, + "total_evaluation_time_seconds": "23943.242170921993" +} \ No newline at end of file diff --git a/lmeval/mistral_aya(hruska)/07_lr2e5_r16_a16/results_2026-10-02T14-27-49.730611.json b/lmeval/mistral_aya(hruska)/07_lr2e5_r16_a16/results_2026-10-02T14-27-49.730611.json new file mode 100644 index 0000000..2637a6b --- /dev/null +++ b/lmeval/mistral_aya(hruska)/07_lr2e5_r16_a16/results_2026-10-02T14-27-49.730611.json @@ -0,0 +1,432 @@ +{ + "results": { + "arc_sk": { + "alias": "arc_sk", + "acc,none": 0.29426860564585117, + "acc_stderr,none": 0.013334301264787742, + "acc_norm,none": 0.32763045337895635, + "acc_norm_stderr,none": 0.013733294316075938 + }, + "hellaswag_sk": { + "alias": "hellaswag_sk", + "acc,none": 0.41560358460727465, + "acc_stderr,none": 0.005060549081722857, + "acc_norm,none": 0.522509225092251, + "acc_norm_stderr,none": 0.005129011836544455 + }, + "m_mmlu_sk": { + "alias": "m_mmlu_sk", + "acc,none": 0.3379268105956209, + "acc_stderr,none": 0.00413881591806207 + }, + "sklegal": { + "alias": "sklegal", + "acc,none": 0.29610194902548725, + "acc_stderr,none": 0.012504330376959024, + "acc_norm,none": 0.47226386806596704, + "acc_norm_stderr,none": 0.013673689066311853 + }, + "skquad": { + "alias": "skquad", + "exact,none": 11.322132943754566, + "exact_stderr,none": "N/A", + "f1,none": 33.83807391933327, + "f1_stderr,none": "N/A", + "HasAns_exact,none": 13.899217848442108, + "HasAns_exact_stderr,none": "N/A", + "HasAns_f1,none": 41.565618972813276, + "HasAns_f1_stderr,none": "N/A", + "NoAns_exact,none": 0.05605381165919283, + "NoAns_exact_stderr,none": "N/A", + "NoAns_f1,none": 0.05605381165919283, + "NoAns_f1_stderr,none": "N/A", + "best_exact,none": 18.61629969738078, + "best_exact_stderr,none": "N/A", + "best_f1,none": 33.95862223270098, + "best_f1_stderr,none": "N/A" + }, + "truthfulqa_sk_mc1": { + "alias": "truthfulqa_sk_mc1", + "acc,none": 0.16709511568123395, + "acc_stderr,none": 0.013383480568197463 + }, + "truthfulqa_sk_mc2": { + "alias": "truthfulqa_sk_mc2", + "acc,none": 0.32463255972324384, + "acc_stderr,none": 0.01427921659123273 + } + }, + "group_subtasks": { + "arc_sk": [], + "hellaswag_sk": [], + "m_mmlu_sk": [], + "sklegal": [], + "skquad": [], + "truthfulqa_sk_mc1": [], + "truthfulqa_sk_mc2": [] + }, + "configs": { + "arc_sk": { + "task": "arc_sk", + "tag": [ + "arc_multilingual" + ], + "dataset_path": "alexandrainst/m_arc", + "dataset_name": "sk", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n # breakpoint()\n out_doc = {\n \"id\": doc[\"id\"],\n \"query\": \"Question: \" + preprocess(doc[\"instruction\"]) + \"\\nAnswer:\",\n \"choices\": [\n preprocess(option)\n for option in [\n doc[\"option_a\"],\n doc[\"option_b\"],\n doc[\"option_c\"],\n doc[\"option_d\"],\n doc[\"option_e\"],\n ]\n if option\n ],\n \"gold\": [\"A\", \"B\", \"C\", \"D\", \"E\"].index(doc[\"answer\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "gold", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "query", + "metadata": { + "version": 2.0 + } + }, + "hellaswag_sk": { + "task": "hellaswag_sk", + "tag": [ + "hellaswag_multilingual" + ], + "dataset_path": "alexandrainst/m_hellaswag", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "{{label.lstrip()}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0 + } + }, + "m_mmlu_sk": { + "task": "m_mmlu_sk", + "tag": [ + "m_mmlu" + ], + "dataset_path": "alexandrainst/m_mmlu", + "dataset_name": "sk", + "test_split": "test", + "fewshot_split": "train", + "doc_to_text": "{{instruction.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:", + "doc_to_target": "answer", + "unsafe_code": false, + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n" + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "sklegal": { + "task": "sklegal", + "dataset_path": "TUKE-KEMT/slovak-legal-exam", + "test_split": "test", + "doc_to_text": "{{prompt}} ", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{answers}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1 + } + }, + "skquad": { + "task": "skquad", + "unsafe_code": false, + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "output_type": "generate_until", + "generation_kwargs": { + "until": [ + "\n\n" + ], + "do_sample": false + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 3 + } + }, + "truthfulqa_sk_mc1": { + "task": "truthfulqa_sk_mc1", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc1_choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + }, + "truthfulqa_sk_mc2": { + "task": "truthfulqa_sk_mc2", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc2_choices", + "process_results": "def process_results_mc2(doc, results):\n lls, is_greedy = zip(*results)\n\n # Split on the first `0` as everything before it is true (`1`).\n split_idx = list(doc[\"mc2_targets\"][\"labels\"]).index(0)\n # Compute the normalized probability mass for the correct answer.\n ll_true, ll_false = lls[:split_idx], lls[split_idx:]\n p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false))\n p_true = p_true / (sum(p_true) + sum(p_false))\n\n return {\"acc\": sum(p_true)}\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + } + }, + "versions": { + "arc_sk": 2.0, + "hellaswag_sk": 1.0, + "m_mmlu_sk": 0.0, + "sklegal": 1, + "skquad": 3, + "truthfulqa_sk_mc1": 1.0, + "truthfulqa_sk_mc2": 1.0 + }, + "n-shot": { + "arc_sk": 0, + "hellaswag_sk": 0, + "m_mmlu_sk": 0, + "sklegal": 0, + "skquad": 0, + "truthfulqa_sk_mc1": 0, + "truthfulqa_sk_mc2": 0 + }, + "higher_is_better": { + "arc_sk": { + "acc": true, + "acc_norm": true + }, + "hellaswag_sk": { + "acc": true, + "acc_norm": true + }, + "m_mmlu_sk": { + "acc": true + }, + "sklegal": { + "acc": true, + "acc_norm": true + }, + "skquad": { + "exact": true, + "f1": true, + "HasAns_exact": true, + "HasAns_f1": true, + "NoAns_exact": true, + "NoAns_f1": true, + "best_exact": true, + "best_f1": true + }, + "truthfulqa_sk_mc1": { + "acc": true + }, + "truthfulqa_sk_mc2": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_sk_mc2": { + "original": 778, + "effective": 778 + }, + "truthfulqa_sk_mc1": { + "original": 778, + "effective": 778 + }, + "skquad": { + "original": 9583, + "effective": 9583 + }, + "sklegal": { + "original": 1334, + "effective": 1334 + }, + "m_mmlu_sk": { + "original": 13062, + "effective": 13062 + }, + "hellaswag_sk": { + "original": 9485, + "effective": 9485 + }, + "arc_sk": { + "original": 1169, + "effective": 1169 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_07_lr2e5_r16_a16,dtype=bfloat16", + "model_num_parameters": 7283675136, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "089497ae0d72018e9591895c54f774bf0b4a83a4", + "peft_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": "6d2abda4", + "date": 1790924397.3269148, + "pretty_env_info": "PyTorch version: 2.6.0+cu124\nIs debug build: False\nCUDA used to build PyTorch: 12.4\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.11.16 (main, Aug 27 2026, 14:44:21) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.8.0-138-generic-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA RTX A4500\nGPU 1: NVIDIA RTX A4500\n\nNvidia driver version: 550.163.01\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 16\nOn-line CPU(s) list: 0-15\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) W-2245 CPU @ 3.90GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 8\nSocket(s): 1\nStepping: 7\nCPU(s) scaling MHz: 27%\nCPU max MHz: 4700.0000\nCPU min MHz: 1200.0000\nBogoMIPS: 7799.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 intel_ppin ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm mpx rdt_a avx512f avx512dq rdseed adx smap clflushopt clwb intel_pt avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512_vnni md_clear flush_l1d arch_capabilities ibpb_exit_to_user\nVirtualization: VT-x\nL1d cache: 256 KiB (8 instances)\nL1i cache: 256 KiB (8 instances)\nL2 cache: 8 MiB (8 instances)\nL3 cache: 16.5 MiB (1 instance)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-15\nVulnerability Gather data sampling: Mitigation; Microcode\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX disabled\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Mitigation; Clear CPU buffers; SMT vulnerable\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced / Automatic IBRS; IBPB conditional; PBRSB-eIBRS SW sequence; BHI SW loop, KVM SW loop\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Mitigation; TSX disabled\nVulnerability Vmscape: Mitigation; IBPB before exit to userspace\n\nVersions of relevant libraries:\n[pip3] numpy==2.4.6\n[pip3] nvidia-cublas-cu12==12.4.5.8\n[pip3] nvidia-cuda-cupti-cu12==12.4.127\n[pip3] nvidia-cuda-nvrtc-cu12==12.4.127\n[pip3] nvidia-cuda-runtime-cu12==12.4.127\n[pip3] nvidia-cudnn-cu12==9.1.0.70\n[pip3] nvidia-cufft-cu12==11.2.1.3\n[pip3] nvidia-curand-cu12==10.3.5.147\n[pip3] nvidia-cusolver-cu12==11.6.1.9\n[pip3] nvidia-cusparse-cu12==12.3.1.170\n[pip3] nvidia-cusparselt-cu12==0.6.2\n[pip3] nvidia-nccl-cu12==2.21.5\n[pip3] nvidia-nvjitlink-cu12==12.4.127\n[pip3] nvidia-nvtx-cu12==12.4.127\n[pip3] torch==2.6.0+cu124\n[pip3] triton==3.2.0\n[conda] numpy 2.4.6 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.4.5.8 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.1.0.70 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.2.1.3 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.5.147 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.6.1.9 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.3.1.170 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.6.2 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.21.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.4.127 pypi_0 pypi\n[conda] torch 2.6.0+cu124 pypi_0 pypi\n[conda] triton 3.2.0 pypi_0 pypi", + "transformers_version": "4.48.2", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "", + "2" + ], + "tokenizer_eos_token": [ + "", + "2" + ], + "tokenizer_bos_token": [ + "", + "1" + ], + "eot_token_id": 2, + "max_length": 32768, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/home/schwarc/diplomovka/hruska_compare9_100k_fixed_07_lr2e5_r16_a16", + "model_name_sanitized": "__home__schwarc__diplomovka__hruska_compare9_100k_fixed_07_lr2e5_r16_a16", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 2761431.340898049, + "end_time": 2788311.530887369, + "total_evaluation_time_seconds": "26880.189989320002" +} \ No newline at end of file diff --git a/lmeval/mistral_aya(hruska)/08_attention_only_r16_a16_lr5e5/results_2026-10-02T19-20-02.720498.json b/lmeval/mistral_aya(hruska)/08_attention_only_r16_a16_lr5e5/results_2026-10-02T19-20-02.720498.json new file mode 100644 index 0000000..aa7ebe2 --- /dev/null +++ b/lmeval/mistral_aya(hruska)/08_attention_only_r16_a16_lr5e5/results_2026-10-02T19-20-02.720498.json @@ -0,0 +1,432 @@ +{ + "results": { + "arc_sk": { + "alias": "arc_sk", + "acc,none": 0.31565440547476475, + "acc_stderr,none": 0.013599478163435343, + "acc_norm,none": 0.35243798118049613, + "acc_norm_stderr,none": 0.013978501429969684 + }, + "hellaswag_sk": { + "alias": "hellaswag_sk", + "acc,none": 0.4157090142329995, + "acc_stderr,none": 0.0050607343573017055, + "acc_norm,none": 0.5217712177121772, + "acc_norm_stderr,none": 0.005129347731280019 + }, + "m_mmlu_sk": { + "alias": "m_mmlu_sk", + "acc,none": 0.3448170264890522, + "acc_stderr,none": 0.004158985802729384 + }, + "sklegal": { + "alias": "sklegal", + "acc,none": 0.2893553223388306, + "acc_stderr,none": 0.012420152167938028, + "acc_norm,none": 0.4767616191904048, + "acc_norm_stderr,none": 0.01367997692560851 + }, + "skquad": { + "alias": "skquad", + "exact,none": 18.136283001147866, + "exact_stderr,none": "N/A", + "f1,none": 43.37517960476381, + "f1_stderr,none": "N/A", + "HasAns_exact,none": 22.28490832157969, + "HasAns_exact_stderr,none": "N/A", + "HasAns_f1,none": 53.29713375464183, + "HasAns_f1_stderr,none": "N/A", + "NoAns_exact,none": 0.0, + "NoAns_exact_stderr,none": "N/A", + "NoAns_f1,none": 0.0, + "NoAns_f1_stderr,none": "N/A", + "best_exact,none": 19.628508817698005, + "best_exact_stderr,none": "N/A", + "best_f1,none": 43.397808641902465, + "best_f1_stderr,none": "N/A" + }, + "truthfulqa_sk_mc1": { + "alias": "truthfulqa_sk_mc1", + "acc,none": 0.18766066838046272, + "acc_stderr,none": 0.014006991356151454 + }, + "truthfulqa_sk_mc2": { + "alias": "truthfulqa_sk_mc2", + "acc,none": 0.3673805917101876, + "acc_stderr,none": 0.014632640494394517 + } + }, + "group_subtasks": { + "arc_sk": [], + "hellaswag_sk": [], + "m_mmlu_sk": [], + "sklegal": [], + "skquad": [], + "truthfulqa_sk_mc1": [], + "truthfulqa_sk_mc2": [] + }, + "configs": { + "arc_sk": { + "task": "arc_sk", + "tag": [ + "arc_multilingual" + ], + "dataset_path": "alexandrainst/m_arc", + "dataset_name": "sk", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n # breakpoint()\n out_doc = {\n \"id\": doc[\"id\"],\n \"query\": \"Question: \" + preprocess(doc[\"instruction\"]) + \"\\nAnswer:\",\n \"choices\": [\n preprocess(option)\n for option in [\n doc[\"option_a\"],\n doc[\"option_b\"],\n doc[\"option_c\"],\n doc[\"option_d\"],\n doc[\"option_e\"],\n ]\n if option\n ],\n \"gold\": [\"A\", \"B\", \"C\", \"D\", \"E\"].index(doc[\"answer\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "gold", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "query", + "metadata": { + "version": 2.0 + } + }, + "hellaswag_sk": { + "task": "hellaswag_sk", + "tag": [ + "hellaswag_multilingual" + ], + "dataset_path": "alexandrainst/m_hellaswag", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "{{label.lstrip()}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0 + } + }, + "m_mmlu_sk": { + "task": "m_mmlu_sk", + "tag": [ + "m_mmlu" + ], + "dataset_path": "alexandrainst/m_mmlu", + "dataset_name": "sk", + "test_split": "test", + "fewshot_split": "train", + "doc_to_text": "{{instruction.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:", + "doc_to_target": "answer", + "unsafe_code": false, + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n" + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "sklegal": { + "task": "sklegal", + "dataset_path": "TUKE-KEMT/slovak-legal-exam", + "test_split": "test", + "doc_to_text": "{{prompt}} ", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{answers}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1 + } + }, + "skquad": { + "task": "skquad", + "unsafe_code": false, + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "output_type": "generate_until", + "generation_kwargs": { + "until": [ + "\n\n" + ], + "do_sample": false + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 3 + } + }, + "truthfulqa_sk_mc1": { + "task": "truthfulqa_sk_mc1", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc1_choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + }, + "truthfulqa_sk_mc2": { + "task": "truthfulqa_sk_mc2", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc2_choices", + "process_results": "def process_results_mc2(doc, results):\n lls, is_greedy = zip(*results)\n\n # Split on the first `0` as everything before it is true (`1`).\n split_idx = list(doc[\"mc2_targets\"][\"labels\"]).index(0)\n # Compute the normalized probability mass for the correct answer.\n ll_true, ll_false = lls[:split_idx], lls[split_idx:]\n p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false))\n p_true = p_true / (sum(p_true) + sum(p_false))\n\n return {\"acc\": sum(p_true)}\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + } + }, + "versions": { + "arc_sk": 2.0, + "hellaswag_sk": 1.0, + "m_mmlu_sk": 0.0, + "sklegal": 1, + "skquad": 3, + "truthfulqa_sk_mc1": 1.0, + "truthfulqa_sk_mc2": 1.0 + }, + "n-shot": { + "arc_sk": 0, + "hellaswag_sk": 0, + "m_mmlu_sk": 0, + "sklegal": 0, + "skquad": 0, + "truthfulqa_sk_mc1": 0, + "truthfulqa_sk_mc2": 0 + }, + "higher_is_better": { + "arc_sk": { + "acc": true, + "acc_norm": true + }, + "hellaswag_sk": { + "acc": true, + "acc_norm": true + }, + "m_mmlu_sk": { + "acc": true + }, + "sklegal": { + "acc": true, + "acc_norm": true + }, + "skquad": { + "exact": true, + "f1": true, + "HasAns_exact": true, + "HasAns_f1": true, + "NoAns_exact": true, + "NoAns_f1": true, + "best_exact": true, + "best_f1": true + }, + "truthfulqa_sk_mc1": { + "acc": true + }, + "truthfulqa_sk_mc2": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_sk_mc2": { + "original": 778, + "effective": 778 + }, + "truthfulqa_sk_mc1": { + "original": 778, + "effective": 778 + }, + "skquad": { + "original": 9583, + "effective": 9583 + }, + "sklegal": { + "original": 1334, + "effective": 1334 + }, + "m_mmlu_sk": { + "original": 13062, + "effective": 13062 + }, + "hellaswag_sk": { + "original": 9485, + "effective": 9485 + }, + "arc_sk": { + "original": 1169, + "effective": 1169 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_08_attention_only_r16_a16_lr5e5,dtype=bfloat16", + "model_num_parameters": 7255363584, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "089497ae0d72018e9591895c54f774bf0b4a83a4", + "peft_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": "6d2abda4", + "date": 1790951302.7615132, + "pretty_env_info": "PyTorch version: 2.6.0+cu124\nIs debug build: False\nCUDA used to build PyTorch: 12.4\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.11.16 (main, Aug 27 2026, 14:44:21) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.8.0-138-generic-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA RTX A4500\nGPU 1: NVIDIA RTX A4500\n\nNvidia driver version: 550.163.01\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 16\nOn-line CPU(s) list: 0-15\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) W-2245 CPU @ 3.90GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 8\nSocket(s): 1\nStepping: 7\nCPU(s) scaling MHz: 27%\nCPU max MHz: 4700.0000\nCPU min MHz: 1200.0000\nBogoMIPS: 7799.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 intel_ppin ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm mpx rdt_a avx512f avx512dq rdseed adx smap clflushopt clwb intel_pt avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512_vnni md_clear flush_l1d arch_capabilities ibpb_exit_to_user\nVirtualization: VT-x\nL1d cache: 256 KiB (8 instances)\nL1i cache: 256 KiB (8 instances)\nL2 cache: 8 MiB (8 instances)\nL3 cache: 16.5 MiB (1 instance)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-15\nVulnerability Gather data sampling: Mitigation; Microcode\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX disabled\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Mitigation; Clear CPU buffers; SMT vulnerable\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced / Automatic IBRS; IBPB conditional; PBRSB-eIBRS SW sequence; BHI SW loop, KVM SW loop\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Mitigation; TSX disabled\nVulnerability Vmscape: Mitigation; IBPB before exit to userspace\n\nVersions of relevant libraries:\n[pip3] numpy==2.4.6\n[pip3] nvidia-cublas-cu12==12.4.5.8\n[pip3] nvidia-cuda-cupti-cu12==12.4.127\n[pip3] nvidia-cuda-nvrtc-cu12==12.4.127\n[pip3] nvidia-cuda-runtime-cu12==12.4.127\n[pip3] nvidia-cudnn-cu12==9.1.0.70\n[pip3] nvidia-cufft-cu12==11.2.1.3\n[pip3] nvidia-curand-cu12==10.3.5.147\n[pip3] nvidia-cusolver-cu12==11.6.1.9\n[pip3] nvidia-cusparse-cu12==12.3.1.170\n[pip3] nvidia-cusparselt-cu12==0.6.2\n[pip3] nvidia-nccl-cu12==2.21.5\n[pip3] nvidia-nvjitlink-cu12==12.4.127\n[pip3] nvidia-nvtx-cu12==12.4.127\n[pip3] torch==2.6.0+cu124\n[pip3] triton==3.2.0\n[conda] numpy 2.4.6 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.4.5.8 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.1.0.70 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.2.1.3 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.5.147 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.6.1.9 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.3.1.170 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.6.2 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.21.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.4.127 pypi_0 pypi\n[conda] torch 2.6.0+cu124 pypi_0 pypi\n[conda] triton 3.2.0 pypi_0 pypi", + "transformers_version": "4.48.2", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "", + "2" + ], + "tokenizer_eos_token": [ + "", + "2" + ], + "tokenizer_bos_token": [ + "", + "1" + ], + "eot_token_id": 2, + "max_length": 32768, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/home/schwarc/diplomovka/hruska_compare9_100k_fixed_08_attention_only_r16_a16_lr5e5", + "model_name_sanitized": "__home__schwarc__diplomovka__hruska_compare9_100k_fixed_08_attention_only_r16_a16_lr5e5", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 2788336.856909314, + "end_time": 2805844.520598981, + "total_evaluation_time_seconds": "17507.663689666893" +} \ No newline at end of file diff --git a/lmeval/mistral_aya(hruska)/09_batch2_accum4_r16_a16_lr5e5/results_2026-10-03T01-58-48.046423.json b/lmeval/mistral_aya(hruska)/09_batch2_accum4_r16_a16_lr5e5/results_2026-10-03T01-58-48.046423.json new file mode 100644 index 0000000..5b326e8 --- /dev/null +++ b/lmeval/mistral_aya(hruska)/09_batch2_accum4_r16_a16_lr5e5/results_2026-10-03T01-58-48.046423.json @@ -0,0 +1,432 @@ +{ + "results": { + "arc_sk": { + "alias": "arc_sk", + "acc,none": 0.29597946963216426, + "acc_stderr,none": 0.0133567880486437, + "acc_norm,none": 0.3378956372968349, + "acc_norm_stderr,none": 0.013839904571755934 + }, + "hellaswag_sk": { + "alias": "hellaswag_sk", + "acc,none": 0.4136004217185029, + "acc_stderr,none": 0.005056983521409346, + "acc_norm,none": 0.5209277807063785, + "acc_norm_stderr,none": 0.005129717875513435 + }, + "m_mmlu_sk": { + "alias": "m_mmlu_sk", + "acc,none": 0.35140101056499773, + "acc_stderr,none": 0.004177355366639571 + }, + "sklegal": { + "alias": "sklegal", + "acc,none": 0.2871064467766117, + "acc_stderr,none": 0.012391353342405943, + "acc_norm,none": 0.46926536731634183, + "acc_norm_stderr,none": 0.013668878751636108 + }, + "skquad": { + "alias": "skquad", + "exact,none": 13.388291766670145, + "exact_stderr,none": "N/A", + "f1,none": 35.3128443275733, + "f1_stderr,none": "N/A", + "HasAns_exact,none": 16.450827029106296, + "HasAns_exact_stderr,none": "N/A", + "HasAns_f1,none": 43.3905612503058, + "HasAns_f1_stderr,none": "N/A", + "NoAns_exact,none": 0.0, + "NoAns_exact_stderr,none": "N/A", + "NoAns_f1,none": 0.0, + "NoAns_f1_stderr,none": "N/A", + "best_exact,none": 18.62673484295106, + "best_exact_stderr,none": "N/A", + "best_f1,none": 35.431132341766855, + "best_f1_stderr,none": "N/A" + }, + "truthfulqa_sk_mc1": { + "alias": "truthfulqa_sk_mc1", + "acc,none": 0.18637532133676094, + "acc_stderr,none": 0.013969978886224855 + }, + "truthfulqa_sk_mc2": { + "alias": "truthfulqa_sk_mc2", + "acc,none": 0.340221777957301, + "acc_stderr,none": 0.014322928653885689 + } + }, + "group_subtasks": { + "arc_sk": [], + "hellaswag_sk": [], + "m_mmlu_sk": [], + "sklegal": [], + "skquad": [], + "truthfulqa_sk_mc1": [], + "truthfulqa_sk_mc2": [] + }, + "configs": { + "arc_sk": { + "task": "arc_sk", + "tag": [ + "arc_multilingual" + ], + "dataset_path": "alexandrainst/m_arc", + "dataset_name": "sk", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n # breakpoint()\n out_doc = {\n \"id\": doc[\"id\"],\n \"query\": \"Question: \" + preprocess(doc[\"instruction\"]) + \"\\nAnswer:\",\n \"choices\": [\n preprocess(option)\n for option in [\n doc[\"option_a\"],\n doc[\"option_b\"],\n doc[\"option_c\"],\n doc[\"option_d\"],\n doc[\"option_e\"],\n ]\n if option\n ],\n \"gold\": [\"A\", \"B\", \"C\", \"D\", \"E\"].index(doc[\"answer\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "gold", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "query", + "metadata": { + "version": 2.0 + } + }, + "hellaswag_sk": { + "task": "hellaswag_sk", + "tag": [ + "hellaswag_multilingual" + ], + "dataset_path": "alexandrainst/m_hellaswag", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "{{label.lstrip()}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0 + } + }, + "m_mmlu_sk": { + "task": "m_mmlu_sk", + "tag": [ + "m_mmlu" + ], + "dataset_path": "alexandrainst/m_mmlu", + "dataset_name": "sk", + "test_split": "test", + "fewshot_split": "train", + "doc_to_text": "{{instruction.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:", + "doc_to_target": "answer", + "unsafe_code": false, + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n" + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "sklegal": { + "task": "sklegal", + "dataset_path": "TUKE-KEMT/slovak-legal-exam", + "test_split": "test", + "doc_to_text": "{{prompt}} ", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{answers}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1 + } + }, + "skquad": { + "task": "skquad", + "unsafe_code": false, + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "output_type": "generate_until", + "generation_kwargs": { + "until": [ + "\n\n" + ], + "do_sample": false + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 3 + } + }, + "truthfulqa_sk_mc1": { + "task": "truthfulqa_sk_mc1", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc1_choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + }, + "truthfulqa_sk_mc2": { + "task": "truthfulqa_sk_mc2", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc2_choices", + "process_results": "def process_results_mc2(doc, results):\n lls, is_greedy = zip(*results)\n\n # Split on the first `0` as everything before it is true (`1`).\n split_idx = list(doc[\"mc2_targets\"][\"labels\"]).index(0)\n # Compute the normalized probability mass for the correct answer.\n ll_true, ll_false = lls[:split_idx], lls[split_idx:]\n p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false))\n p_true = p_true / (sum(p_true) + sum(p_false))\n\n return {\"acc\": sum(p_true)}\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + } + }, + "versions": { + "arc_sk": 2.0, + "hellaswag_sk": 1.0, + "m_mmlu_sk": 0.0, + "sklegal": 1, + "skquad": 3, + "truthfulqa_sk_mc1": 1.0, + "truthfulqa_sk_mc2": 1.0 + }, + "n-shot": { + "arc_sk": 0, + "hellaswag_sk": 0, + "m_mmlu_sk": 0, + "sklegal": 0, + "skquad": 0, + "truthfulqa_sk_mc1": 0, + "truthfulqa_sk_mc2": 0 + }, + "higher_is_better": { + "arc_sk": { + "acc": true, + "acc_norm": true + }, + "hellaswag_sk": { + "acc": true, + "acc_norm": true + }, + "m_mmlu_sk": { + "acc": true + }, + "sklegal": { + "acc": true, + "acc_norm": true + }, + "skquad": { + "exact": true, + "f1": true, + "HasAns_exact": true, + "HasAns_f1": true, + "NoAns_exact": true, + "NoAns_f1": true, + "best_exact": true, + "best_f1": true + }, + "truthfulqa_sk_mc1": { + "acc": true + }, + "truthfulqa_sk_mc2": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_sk_mc2": { + "original": 778, + "effective": 778 + }, + "truthfulqa_sk_mc1": { + "original": 778, + "effective": 778 + }, + "skquad": { + "original": 9583, + "effective": 9583 + }, + "sklegal": { + "original": 1334, + "effective": 1334 + }, + "m_mmlu_sk": { + "original": 13062, + "effective": 13062 + }, + "hellaswag_sk": { + "original": 9485, + "effective": 9485 + }, + "arc_sk": { + "original": 1169, + "effective": 1169 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_09_batch2_accum4_r16_a16_lr5e5,dtype=bfloat16", + "model_num_parameters": 7283675136, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "089497ae0d72018e9591895c54f774bf0b4a83a4", + "peft_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": "6d2abda4", + "date": 1790968835.5757186, + "pretty_env_info": "PyTorch version: 2.6.0+cu124\nIs debug build: False\nCUDA used to build PyTorch: 12.4\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.11.16 (main, Aug 27 2026, 14:44:21) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.8.0-138-generic-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA RTX A4500\nGPU 1: NVIDIA RTX A4500\n\nNvidia driver version: 550.163.01\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 16\nOn-line CPU(s) list: 0-15\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) W-2245 CPU @ 3.90GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 8\nSocket(s): 1\nStepping: 7\nCPU(s) scaling MHz: 26%\nCPU max MHz: 4700.0000\nCPU min MHz: 1200.0000\nBogoMIPS: 7799.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 intel_ppin ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm mpx rdt_a avx512f avx512dq rdseed adx smap clflushopt clwb intel_pt avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512_vnni md_clear flush_l1d arch_capabilities ibpb_exit_to_user\nVirtualization: VT-x\nL1d cache: 256 KiB (8 instances)\nL1i cache: 256 KiB (8 instances)\nL2 cache: 8 MiB (8 instances)\nL3 cache: 16.5 MiB (1 instance)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-15\nVulnerability Gather data sampling: Mitigation; Microcode\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX disabled\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Mitigation; Clear CPU buffers; SMT vulnerable\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced / Automatic IBRS; IBPB conditional; PBRSB-eIBRS SW sequence; BHI SW loop, KVM SW loop\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Mitigation; TSX disabled\nVulnerability Vmscape: Mitigation; IBPB before exit to userspace\n\nVersions of relevant libraries:\n[pip3] numpy==2.4.6\n[pip3] nvidia-cublas-cu12==12.4.5.8\n[pip3] nvidia-cuda-cupti-cu12==12.4.127\n[pip3] nvidia-cuda-nvrtc-cu12==12.4.127\n[pip3] nvidia-cuda-runtime-cu12==12.4.127\n[pip3] nvidia-cudnn-cu12==9.1.0.70\n[pip3] nvidia-cufft-cu12==11.2.1.3\n[pip3] nvidia-curand-cu12==10.3.5.147\n[pip3] nvidia-cusolver-cu12==11.6.1.9\n[pip3] nvidia-cusparse-cu12==12.3.1.170\n[pip3] nvidia-cusparselt-cu12==0.6.2\n[pip3] nvidia-nccl-cu12==2.21.5\n[pip3] nvidia-nvjitlink-cu12==12.4.127\n[pip3] nvidia-nvtx-cu12==12.4.127\n[pip3] torch==2.6.0+cu124\n[pip3] triton==3.2.0\n[conda] numpy 2.4.6 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.4.5.8 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.1.0.70 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.2.1.3 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.5.147 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.6.1.9 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.3.1.170 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.6.2 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.21.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.4.127 pypi_0 pypi\n[conda] torch 2.6.0+cu124 pypi_0 pypi\n[conda] triton 3.2.0 pypi_0 pypi", + "transformers_version": "4.48.2", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "", + "2" + ], + "tokenizer_eos_token": [ + "", + "2" + ], + "tokenizer_bos_token": [ + "", + "1" + ], + "eot_token_id": 2, + "max_length": 32768, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/home/schwarc/diplomovka/hruska_compare9_100k_fixed_09_batch2_accum4_r16_a16_lr5e5", + "model_name_sanitized": "__home__schwarc__diplomovka__hruska_compare9_100k_fixed_09_batch2_accum4_r16_a16_lr5e5", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 2805869.743832301, + "end_time": 2829769.846569304, + "total_evaluation_time_seconds": "23900.102737003006" +} \ No newline at end of file diff --git a/lmeval/mistral_aya(hruska)/10_r16_a32_lr1e4/results_2026-10-03T07-58-30.464881.json b/lmeval/mistral_aya(hruska)/10_r16_a32_lr1e4/results_2026-10-03T07-58-30.464881.json new file mode 100644 index 0000000..732c045 --- /dev/null +++ b/lmeval/mistral_aya(hruska)/10_r16_a32_lr1e4/results_2026-10-03T07-58-30.464881.json @@ -0,0 +1,432 @@ +{ + "results": { + "arc_sk": { + "alias": "arc_sk", + "acc,none": 0.3028229255774166, + "acc_stderr,none": 0.013444495274526859, + "acc_norm,none": 0.32934131736526945, + "acc_norm_stderr,none": 0.013751575689336031 + }, + "hellaswag_sk": { + "alias": "hellaswag_sk", + "acc,none": 0.41012124406958356, + "acc_stderr,none": 0.0050505856239358915, + "acc_norm,none": 0.515656299420137, + "acc_norm_stderr,none": 0.005131699520760396 + }, + "m_mmlu_sk": { + "alias": "m_mmlu_sk", + "acc,none": 0.3448170264890522, + "acc_stderr,none": 0.0041589858027293826 + }, + "sklegal": { + "alias": "sklegal", + "acc,none": 0.2698650674662669, + "acc_stderr,none": 0.012157935205777518, + "acc_norm,none": 0.4617691154422789, + "acc_norm_stderr,none": 0.01365468462479103 + }, + "skquad": { + "alias": "skquad", + "exact,none": 14.160492538870917, + "exact_stderr,none": "N/A", + "f1,none": 37.75612784500817, + "f1_stderr,none": "N/A", + "HasAns_exact,none": 17.38684446723939, + "HasAns_exact_stderr,none": "N/A", + "HasAns_f1,none": 46.37991705843228, + "HasAns_f1_stderr,none": "N/A", + "NoAns_exact,none": 0.05605381165919283, + "NoAns_exact_stderr,none": "N/A", + "NoAns_f1,none": 0.05605381165919283, + "NoAns_f1_stderr,none": "N/A", + "best_exact,none": 18.77282688093499, + "best_exact_stderr,none": "N/A", + "best_f1,none": 38.08192856860334, + "best_f1_stderr,none": "N/A" + }, + "truthfulqa_sk_mc1": { + "alias": "truthfulqa_sk_mc1", + "acc,none": 0.20565552699228792, + "acc_stderr,none": 0.014499870465396263 + }, + "truthfulqa_sk_mc2": { + "alias": "truthfulqa_sk_mc2", + "acc,none": 0.34795284436431645, + "acc_stderr,none": 0.014346130348509813 + } + }, + "group_subtasks": { + "arc_sk": [], + "hellaswag_sk": [], + "m_mmlu_sk": [], + "sklegal": [], + "skquad": [], + "truthfulqa_sk_mc1": [], + "truthfulqa_sk_mc2": [] + }, + "configs": { + "arc_sk": { + "task": "arc_sk", + "tag": [ + "arc_multilingual" + ], + "dataset_path": "alexandrainst/m_arc", + "dataset_name": "sk", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n # breakpoint()\n out_doc = {\n \"id\": doc[\"id\"],\n \"query\": \"Question: \" + preprocess(doc[\"instruction\"]) + \"\\nAnswer:\",\n \"choices\": [\n preprocess(option)\n for option in [\n doc[\"option_a\"],\n doc[\"option_b\"],\n doc[\"option_c\"],\n doc[\"option_d\"],\n doc[\"option_e\"],\n ]\n if option\n ],\n \"gold\": [\"A\", \"B\", \"C\", \"D\", \"E\"].index(doc[\"answer\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "gold", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "query", + "metadata": { + "version": 2.0 + } + }, + "hellaswag_sk": { + "task": "hellaswag_sk", + "tag": [ + "hellaswag_multilingual" + ], + "dataset_path": "alexandrainst/m_hellaswag", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "{{label.lstrip()}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0 + } + }, + "m_mmlu_sk": { + "task": "m_mmlu_sk", + "tag": [ + "m_mmlu" + ], + "dataset_path": "alexandrainst/m_mmlu", + "dataset_name": "sk", + "test_split": "test", + "fewshot_split": "train", + "doc_to_text": "{{instruction.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:", + "doc_to_target": "answer", + "unsafe_code": false, + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n" + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "sklegal": { + "task": "sklegal", + "dataset_path": "TUKE-KEMT/slovak-legal-exam", + "test_split": "test", + "doc_to_text": "{{prompt}} ", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{answers}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1 + } + }, + "skquad": { + "task": "skquad", + "unsafe_code": false, + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "output_type": "generate_until", + "generation_kwargs": { + "until": [ + "\n\n" + ], + "do_sample": false + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 3 + } + }, + "truthfulqa_sk_mc1": { + "task": "truthfulqa_sk_mc1", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc1_choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + }, + "truthfulqa_sk_mc2": { + "task": "truthfulqa_sk_mc2", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc2_choices", + "process_results": "def process_results_mc2(doc, results):\n lls, is_greedy = zip(*results)\n\n # Split on the first `0` as everything before it is true (`1`).\n split_idx = list(doc[\"mc2_targets\"][\"labels\"]).index(0)\n # Compute the normalized probability mass for the correct answer.\n ll_true, ll_false = lls[:split_idx], lls[split_idx:]\n p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false))\n p_true = p_true / (sum(p_true) + sum(p_false))\n\n return {\"acc\": sum(p_true)}\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + } + }, + "versions": { + "arc_sk": 2.0, + "hellaswag_sk": 1.0, + "m_mmlu_sk": 0.0, + "sklegal": 1, + "skquad": 3, + "truthfulqa_sk_mc1": 1.0, + "truthfulqa_sk_mc2": 1.0 + }, + "n-shot": { + "arc_sk": 0, + "hellaswag_sk": 0, + "m_mmlu_sk": 0, + "sklegal": 0, + "skquad": 0, + "truthfulqa_sk_mc1": 0, + "truthfulqa_sk_mc2": 0 + }, + "higher_is_better": { + "arc_sk": { + "acc": true, + "acc_norm": true + }, + "hellaswag_sk": { + "acc": true, + "acc_norm": true + }, + "m_mmlu_sk": { + "acc": true + }, + "sklegal": { + "acc": true, + "acc_norm": true + }, + "skquad": { + "exact": true, + "f1": true, + "HasAns_exact": true, + "HasAns_f1": true, + "NoAns_exact": true, + "NoAns_f1": true, + "best_exact": true, + "best_f1": true + }, + "truthfulqa_sk_mc1": { + "acc": true + }, + "truthfulqa_sk_mc2": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_sk_mc2": { + "original": 778, + "effective": 778 + }, + "truthfulqa_sk_mc1": { + "original": 778, + "effective": 778 + }, + "skquad": { + "original": 9583, + "effective": 9583 + }, + "sklegal": { + "original": 1334, + "effective": 1334 + }, + "m_mmlu_sk": { + "original": 13062, + "effective": 13062 + }, + "hellaswag_sk": { + "original": 9485, + "effective": 9485 + }, + "arc_sk": { + "original": 1169, + "effective": 1169 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/hruska_compare9_100k_fixed_10_r16_a32_lr1e4,dtype=bfloat16", + "model_num_parameters": 7283675136, + "model_dtype": "torch.bfloat16", + "model_revision": "main", + "model_sha": "089497ae0d72018e9591895c54f774bf0b4a83a4", + "peft_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": "6d2abda4", + "date": 1790992760.9371092, + "pretty_env_info": "PyTorch version: 2.6.0+cu124\nIs debug build: False\nCUDA used to build PyTorch: 12.4\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 24.04.4 LTS (x86_64)\nGCC version: (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0\nClang version: Could not collect\nCMake version: version 3.28.3\nLibc version: glibc-2.39\n\nPython version: 3.11.16 (main, Aug 27 2026, 14:44:21) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-6.8.0-138-generic-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA RTX A4500\nGPU 1: NVIDIA RTX A4500\n\nNvidia driver version: 550.163.01\ncuDNN version: Could not collect\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 46 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 16\nOn-line CPU(s) list: 0-15\nVendor ID: GenuineIntel\nModel name: Intel(R) Xeon(R) W-2245 CPU @ 3.90GHz\nCPU family: 6\nModel: 85\nThread(s) per core: 2\nCore(s) per socket: 8\nSocket(s): 1\nStepping: 7\nCPU(s) scaling MHz: 27%\nCPU max MHz: 4700.0000\nCPU min MHz: 1200.0000\nBogoMIPS: 7799.87\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 intel_ppin ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm mpx rdt_a avx512f avx512dq rdseed adx smap clflushopt clwb intel_pt avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512_vnni md_clear flush_l1d arch_capabilities ibpb_exit_to_user\nVirtualization: VT-x\nL1d cache: 256 KiB (8 instances)\nL1i cache: 256 KiB (8 instances)\nL2 cache: 8 MiB (8 instances)\nL3 cache: 16.5 MiB (1 instance)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-15\nVulnerability Gather data sampling: Mitigation; Microcode\nVulnerability Indirect target selection: Mitigation; Aligned branch/return thunks\nVulnerability Itlb multihit: KVM: Mitigation: VMX disabled\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Mitigation; Clear CPU buffers; SMT vulnerable\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; Enhanced IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Enhanced / Automatic IBRS; IBPB conditional; PBRSB-eIBRS SW sequence; BHI SW loop, KVM SW loop\nVulnerability Srbds: Not affected\nVulnerability Tsa: Not affected\nVulnerability Tsx async abort: Mitigation; TSX disabled\nVulnerability Vmscape: Mitigation; IBPB before exit to userspace\n\nVersions of relevant libraries:\n[pip3] numpy==2.4.6\n[pip3] nvidia-cublas-cu12==12.4.5.8\n[pip3] nvidia-cuda-cupti-cu12==12.4.127\n[pip3] nvidia-cuda-nvrtc-cu12==12.4.127\n[pip3] nvidia-cuda-runtime-cu12==12.4.127\n[pip3] nvidia-cudnn-cu12==9.1.0.70\n[pip3] nvidia-cufft-cu12==11.2.1.3\n[pip3] nvidia-curand-cu12==10.3.5.147\n[pip3] nvidia-cusolver-cu12==11.6.1.9\n[pip3] nvidia-cusparse-cu12==12.3.1.170\n[pip3] nvidia-cusparselt-cu12==0.6.2\n[pip3] nvidia-nccl-cu12==2.21.5\n[pip3] nvidia-nvjitlink-cu12==12.4.127\n[pip3] nvidia-nvtx-cu12==12.4.127\n[pip3] torch==2.6.0+cu124\n[pip3] triton==3.2.0\n[conda] numpy 2.4.6 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.4.5.8 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.1.0.70 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.2.1.3 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.5.147 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.6.1.9 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.3.1.170 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.6.2 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.21.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.4.127 pypi_0 pypi\n[conda] torch 2.6.0+cu124 pypi_0 pypi\n[conda] triton 3.2.0 pypi_0 pypi", + "transformers_version": "4.48.2", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "", + "2" + ], + "tokenizer_eos_token": [ + "", + "2" + ], + "tokenizer_bos_token": [ + "", + "1" + ], + "eot_token_id": 2, + "max_length": 32768, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/home/schwarc/diplomovka/hruska_compare9_100k_fixed_10_r16_a32_lr1e4", + "model_name_sanitized": "__home__schwarc__diplomovka__hruska_compare9_100k_fixed_10_r16_a32_lr1e4", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 2829795.079359318, + "end_time": 2851352.265166676, + "total_evaluation_time_seconds": "21557.185807358008" +} \ No newline at end of file diff --git a/lmeval/mistral_aya(titan)/titan_aya_25k/results_2026-10-01T21-46-30.546755.json b/lmeval/mistral_aya(titan)/titan_aya_25k/results_2026-10-01T21-46-30.546755.json new file mode 100644 index 0000000..27350b1 --- /dev/null +++ b/lmeval/mistral_aya(titan)/titan_aya_25k/results_2026-10-01T21-46-30.546755.json @@ -0,0 +1,432 @@ +{ + "results": { + "arc_sk": { + "alias": "arc_sk", + "acc,none": 0.2643284858853721, + "acc_stderr,none": 0.01290305453359847, + "acc_norm,none": 0.32506415739948674, + "acc_norm_stderr,none": 0.013705483696967787 + }, + "hellaswag_sk": { + "alias": "hellaswag_sk", + "acc,none": 0.4048497627833421, + "acc_stderr,none": 0.005040393901571885, + "acc_norm,none": 0.4942540853979968, + "acc_norm_stderr,none": 0.005133878104632753 + }, + "m_mmlu_sk": { + "alias": "m_mmlu_sk", + "acc,none": 0.3341754708314194, + "acc_stderr,none": 0.004127422857128726 + }, + "sklegal": { + "alias": "sklegal", + "acc,none": 0.2653673163418291, + "acc_stderr,none": 0.012093270426850032, + "acc_norm,none": 0.4587706146926537, + "acc_norm_stderr,none": 0.013648138014329935 + }, + "skquad": { + "alias": "skquad", + "exact,none": 4.633204633204633, + "exact_stderr,none": "N/A", + "f1,none": 25.867828563515147, + "f1_stderr,none": "N/A", + "HasAns_exact,none": 5.667393255545583, + "HasAns_exact_stderr,none": "N/A", + "HasAns_f1,none": 31.759379551758638, + "HasAns_f1_stderr,none": "N/A", + "NoAns_exact,none": 0.11210762331838565, + "NoAns_exact_stderr,none": "N/A", + "NoAns_f1,none": 0.11210762331838565, + "NoAns_f1_stderr,none": "N/A", + "best_exact,none": 18.61629969738078, + "best_exact_stderr,none": "N/A", + "best_f1,none": 25.922669866068382, + "best_f1_stderr,none": "N/A" + }, + "truthfulqa_sk_mc1": { + "alias": "truthfulqa_sk_mc1", + "acc,none": 0.17609254498714652, + "acc_stderr,none": 0.013664671382760684 + }, + "truthfulqa_sk_mc2": { + "alias": "truthfulqa_sk_mc2", + "acc,none": 0.32507560661819124, + "acc_stderr,none": 0.014118212074933533 + } + }, + "group_subtasks": { + "arc_sk": [], + "hellaswag_sk": [], + "m_mmlu_sk": [], + "sklegal": [], + "skquad": [], + "truthfulqa_sk_mc1": [], + "truthfulqa_sk_mc2": [] + }, + "configs": { + "arc_sk": { + "task": "arc_sk", + "tag": [ + "arc_multilingual" + ], + "dataset_path": "alexandrainst/m_arc", + "dataset_name": "sk", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n # breakpoint()\n out_doc = {\n \"id\": doc[\"id\"],\n \"query\": \"Question: \" + preprocess(doc[\"instruction\"]) + \"\\nAnswer:\",\n \"choices\": [\n preprocess(option)\n for option in [\n doc[\"option_a\"],\n doc[\"option_b\"],\n doc[\"option_c\"],\n doc[\"option_d\"],\n doc[\"option_e\"],\n ]\n if option\n ],\n \"gold\": [\"A\", \"B\", \"C\", \"D\", \"E\"].index(doc[\"answer\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "gold", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "query", + "metadata": { + "version": 2.0 + } + }, + "hellaswag_sk": { + "task": "hellaswag_sk", + "tag": [ + "hellaswag_multilingual" + ], + "dataset_path": "alexandrainst/m_hellaswag", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "{{label.lstrip()}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0 + } + }, + "m_mmlu_sk": { + "task": "m_mmlu_sk", + "tag": [ + "m_mmlu" + ], + "dataset_path": "alexandrainst/m_mmlu", + "dataset_name": "sk", + "test_split": "test", + "fewshot_split": "train", + "doc_to_text": "{{instruction.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:", + "doc_to_target": "answer", + "unsafe_code": false, + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n" + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "sklegal": { + "task": "sklegal", + "dataset_path": "TUKE-KEMT/slovak-legal-exam", + "test_split": "test", + "doc_to_text": "{{prompt}} ", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{answers}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1 + } + }, + "skquad": { + "task": "skquad", + "unsafe_code": false, + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "output_type": "generate_until", + "generation_kwargs": { + "until": [ + "\n\n" + ], + "do_sample": false + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 3 + } + }, + "truthfulqa_sk_mc1": { + "task": "truthfulqa_sk_mc1", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc1_choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + }, + "truthfulqa_sk_mc2": { + "task": "truthfulqa_sk_mc2", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc2_choices", + "process_results": "def process_results_mc2(doc, results):\n lls, is_greedy = zip(*results)\n\n # Split on the first `0` as everything before it is true (`1`).\n split_idx = list(doc[\"mc2_targets\"][\"labels\"]).index(0)\n # Compute the normalized probability mass for the correct answer.\n ll_true, ll_false = lls[:split_idx], lls[split_idx:]\n p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false))\n p_true = p_true / (sum(p_true) + sum(p_false))\n\n return {\"acc\": sum(p_true)}\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + } + }, + "versions": { + "arc_sk": 2.0, + "hellaswag_sk": 1.0, + "m_mmlu_sk": 0.0, + "sklegal": 1, + "skquad": 3, + "truthfulqa_sk_mc1": 1.0, + "truthfulqa_sk_mc2": 1.0 + }, + "n-shot": { + "arc_sk": 0, + "hellaswag_sk": 0, + "m_mmlu_sk": 0, + "sklegal": 0, + "skquad": 0, + "truthfulqa_sk_mc1": 0, + "truthfulqa_sk_mc2": 0 + }, + "higher_is_better": { + "arc_sk": { + "acc": true, + "acc_norm": true + }, + "hellaswag_sk": { + "acc": true, + "acc_norm": true + }, + "m_mmlu_sk": { + "acc": true + }, + "sklegal": { + "acc": true, + "acc_norm": true + }, + "skquad": { + "exact": true, + "f1": true, + "HasAns_exact": true, + "HasAns_f1": true, + "NoAns_exact": true, + "NoAns_f1": true, + "best_exact": true, + "best_f1": true + }, + "truthfulqa_sk_mc1": { + "acc": true + }, + "truthfulqa_sk_mc2": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_sk_mc2": { + "original": 778, + "effective": 778 + }, + "truthfulqa_sk_mc1": { + "original": 778, + "effective": 778 + }, + "skquad": { + "original": 9583, + "effective": 9583 + }, + "sklegal": { + "original": 1334, + "effective": 1334 + }, + "m_mmlu_sk": { + "original": 13062, + "effective": 13062 + }, + "hellaswag_sk": { + "original": 9485, + "effective": 9485 + }, + "arc_sk": { + "original": 1169, + "effective": 1169 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/saved_checkpoints/checkpoint-25000_titan_aya_lora,dtype=float16", + "model_num_parameters": 7283675136, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "089497ae0d72018e9591895c54f774bf0b4a83a4", + "peft_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": "53504a96", + "date": 1790833148.9662633, + "pretty_env_info": "PyTorch version: 2.6.0+cu124\nIs debug build: False\nCUDA used to build PyTorch: 12.4\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 20.04.6 LTS (x86_64)\nGCC version: (Ubuntu 9.4.0-1ubuntu1~20.04.2) 9.4.0\nClang version: Could not collect\nCMake version: version 3.16.3\nLibc version: glibc-2.31\n\nPython version: 3.11.16 (main, Aug 27 2026, 14:44:21) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-5.15.0-119-generic-x86_64-with-glibc2.31\nIs CUDA available: True\nCUDA runtime version: 10.1.243\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA TITAN RTX\nGPU 1: NVIDIA TITAN RTX\n\nNvidia driver version: 575.57.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.10.2\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 20\nOn-line CPU(s) list: 0-19\nThread(s) per core: 2\nCore(s) per socket: 10\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Core(TM) i9-9820X CPU @ 3.30GHz\nStepping: 4\nCPU MHz: 3300.000\nCPU max MHz: 4200,0000\nCPU min MHz: 1200,0000\nBogoMIPS: 6599.98\nL1d cache: 320 KiB\nL1i cache: 320 KiB\nL2 cache: 10 MiB\nL3 cache: 16,5 MiB\nNUMA node0 CPU(s): 0-19\nVulnerability Gather data sampling: Mitigation; Microcode\nVulnerability Itlb multihit: KVM: Mitigation: VMX unsupported\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Mitigation; Clear CPU buffers; SMT vulnerable\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Mmio stale data: Mitigation; Clear CPU buffers; SMT vulnerable\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; IBRS; IBPB conditional; STIBP conditional; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Mitigation; Clear CPU buffers; SMT vulnerable\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 invpcid_single pti ssbd mba ibrs ibpb stibp fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm cqm mpx rdt_a avx512f avx512dq rdseed adx smap clflushopt clwb intel_pt avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req md_clear flush_l1d arch_capabilities\n\nVersions of relevant libraries:\n[pip3] numpy==2.4.6\n[pip3] nvidia-cublas-cu12==12.4.5.8\n[pip3] nvidia-cuda-cupti-cu12==12.4.127\n[pip3] nvidia-cuda-nvrtc-cu12==12.4.127\n[pip3] nvidia-cuda-runtime-cu12==12.4.127\n[pip3] nvidia-cudnn-cu12==9.1.0.70\n[pip3] nvidia-cufft-cu12==11.2.1.3\n[pip3] nvidia-curand-cu12==10.3.5.147\n[pip3] nvidia-cusolver-cu12==11.6.1.9\n[pip3] nvidia-cusparse-cu12==12.3.1.170\n[pip3] nvidia-cusparselt-cu12==0.6.2\n[pip3] nvidia-nccl-cu12==2.21.5\n[pip3] nvidia-nvjitlink-cu12==12.4.127\n[pip3] nvidia-nvtx-cu12==12.4.127\n[pip3] torch==2.6.0+cu124\n[pip3] triton==3.2.0\n[conda] numpy 2.4.6 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.4.5.8 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.1.0.70 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.2.1.3 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.5.147 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.6.1.9 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.3.1.170 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.6.2 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.21.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.4.127 pypi_0 pypi\n[conda] torch 2.6.0+cu124 pypi_0 pypi\n[conda] triton 3.2.0 pypi_0 pypi", + "transformers_version": "4.48.2", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "", + "2" + ], + "tokenizer_eos_token": [ + "", + "2" + ], + "tokenizer_bos_token": [ + "", + "1" + ], + "eot_token_id": 2, + "max_length": 32768, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/home/schwarc/diplomovka/saved_checkpoints/checkpoint-25000_titan_aya_lora", + "model_name_sanitized": "__home__schwarc__diplomovka__saved_checkpoints__checkpoint-25000_titan_aya_lora", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 6735750.281901046, + "end_time": 6786600.251450867, + "total_evaluation_time_seconds": "50849.96954982169" +} \ No newline at end of file diff --git a/lmeval/mistral_aya(titan)/titan_aya_full/results_2026-10-01T01-37-58.346736.json b/lmeval/mistral_aya(titan)/titan_aya_full/results_2026-10-01T01-37-58.346736.json new file mode 100644 index 0000000..fd501d9 --- /dev/null +++ b/lmeval/mistral_aya(titan)/titan_aya_full/results_2026-10-01T01-37-58.346736.json @@ -0,0 +1,432 @@ +{ + "results": { + "arc_sk": { + "alias": "arc_sk", + "acc,none": 0.30025662959794697, + "acc_stderr,none": 0.01341202262936666, + "acc_norm,none": 0.34473909324208724, + "acc_norm_stderr,none": 0.013906920607432552 + }, + "hellaswag_sk": { + "alias": "hellaswag_sk", + "acc,none": 0.39768054823405374, + "acc_stderr,none": 0.005025564441276893, + "acc_norm,none": 0.4792830785450712, + "acc_norm_stderr,none": 0.005129808122680316 + }, + "m_mmlu_sk": { + "alias": "m_mmlu_sk", + "acc,none": 0.2719338539274231, + "acc_stderr,none": 0.0038933983713266575 + }, + "sklegal": { + "alias": "sklegal", + "acc,none": 0.25337331334332835, + "acc_stderr,none": 0.011912889544770117, + "acc_norm,none": 0.43928035982008995, + "acc_norm_stderr,none": 0.013593418967725883 + }, + "skquad": { + "alias": "skquad", + "exact,none": 10.883856829802776, + "exact_stderr,none": "N/A", + "f1,none": 35.04710724582597, + "f1_stderr,none": "N/A", + "HasAns_exact,none": 13.373509424285166, + "HasAns_exact_stderr,none": "N/A", + "HasAns_f1,none": 43.0640375351648, + "HasAns_f1_stderr,none": "N/A", + "NoAns_exact,none": 0.0, + "NoAns_exact_stderr,none": "N/A", + "NoAns_f1,none": 0.0, + "NoAns_f1_stderr,none": "N/A", + "best_exact,none": 18.64760513409162, + "best_exact_stderr,none": "N/A", + "best_f1,none": 35.17783479678552, + "best_f1_stderr,none": "N/A" + }, + "truthfulqa_sk_mc1": { + "alias": "truthfulqa_sk_mc1", + "acc,none": 0.20051413881748073, + "acc_stderr,none": 0.01436373503333316 + }, + "truthfulqa_sk_mc2": { + "alias": "truthfulqa_sk_mc2", + "acc,none": 0.36134935010431024, + "acc_stderr,none": 0.014386187990125605 + } + }, + "group_subtasks": { + "arc_sk": [], + "hellaswag_sk": [], + "m_mmlu_sk": [], + "sklegal": [], + "skquad": [], + "truthfulqa_sk_mc1": [], + "truthfulqa_sk_mc2": [] + }, + "configs": { + "arc_sk": { + "task": "arc_sk", + "tag": [ + "arc_multilingual" + ], + "dataset_path": "alexandrainst/m_arc", + "dataset_name": "sk", + "training_split": "train", + "validation_split": "validation", + "test_split": "test", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n # breakpoint()\n out_doc = {\n \"id\": doc[\"id\"],\n \"query\": \"Question: \" + preprocess(doc[\"instruction\"]) + \"\\nAnswer:\",\n \"choices\": [\n preprocess(option)\n for option in [\n doc[\"option_a\"],\n doc[\"option_b\"],\n doc[\"option_c\"],\n doc[\"option_d\"],\n doc[\"option_e\"],\n ]\n if option\n ],\n \"gold\": [\"A\", \"B\", \"C\", \"D\", \"E\"].index(doc[\"answer\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "gold", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "query", + "metadata": { + "version": 2.0 + } + }, + "hellaswag_sk": { + "task": "hellaswag_sk", + "tag": [ + "hellaswag_multilingual" + ], + "dataset_path": "alexandrainst/m_hellaswag", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": "{{label.lstrip()}}", + "unsafe_code": false, + "doc_to_choice": "choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1.0 + } + }, + "m_mmlu_sk": { + "task": "m_mmlu_sk", + "tag": [ + "m_mmlu" + ], + "dataset_path": "alexandrainst/m_mmlu", + "dataset_name": "sk", + "test_split": "test", + "fewshot_split": "train", + "doc_to_text": "{{instruction.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:", + "doc_to_target": "answer", + "unsafe_code": false, + "doc_to_choice": [ + "A", + "B", + "C", + "D" + ], + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "fewshot_config": { + "sampler": "first_n" + }, + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 0.0 + } + }, + "sklegal": { + "task": "sklegal", + "dataset_path": "TUKE-KEMT/slovak-legal-exam", + "test_split": "test", + "doc_to_text": "{{prompt}} ", + "doc_to_target": "label", + "unsafe_code": false, + "doc_to_choice": "{{answers}}", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + }, + { + "metric": "acc_norm", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 1 + } + }, + "skquad": { + "task": "skquad", + "unsafe_code": false, + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "output_type": "generate_until", + "generation_kwargs": { + "until": [ + "\n\n" + ], + "do_sample": false + }, + "repeats": 1, + "should_decontaminate": false, + "metadata": { + "version": 3 + } + }, + "truthfulqa_sk_mc1": { + "task": "truthfulqa_sk_mc1", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc1_choices", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + }, + "truthfulqa_sk_mc2": { + "task": "truthfulqa_sk_mc2", + "tag": [ + "truthfulqa_multilingual" + ], + "dataset_path": "alexandrainst/m_truthfulqa", + "dataset_name": "sk", + "validation_split": "val", + "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n out_doc = {\n \"question\": preprocess(doc[\"question\"]),\n \"query\": QA_PROMPT + \"\\n\\nQ: \" + preprocess(doc[\"question\"]) + \"\\nA:\",\n \"mc1_choices\": doc[\"mc1_targets_choices\"],\n \"mc2_choices\": doc[\"mc2_targets_choices\"],\n \"mc2_targets\": {\"labels\": doc[\"mc2_targets_labels\"]},\n \"gold\": \" \",\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", + "doc_to_text": "query", + "doc_to_target": 0, + "unsafe_code": false, + "doc_to_choice": "mc2_choices", + "process_results": "def process_results_mc2(doc, results):\n lls, is_greedy = zip(*results)\n\n # Split on the first `0` as everything before it is true (`1`).\n split_idx = list(doc[\"mc2_targets\"][\"labels\"]).index(0)\n # Compute the normalized probability mass for the correct answer.\n ll_true, ll_false = lls[:split_idx], lls[split_idx:]\n p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false))\n p_true = p_true / (sum(p_true) + sum(p_false))\n\n return {\"acc\": sum(p_true)}\n", + "description": "", + "target_delimiter": " ", + "fewshot_delimiter": "\n\n", + "num_fewshot": 0, + "metric_list": [ + { + "metric": "acc", + "aggregation": "mean", + "higher_is_better": true + } + ], + "output_type": "multiple_choice", + "repeats": 1, + "should_decontaminate": true, + "doc_to_decontamination_query": "question", + "metadata": { + "version": 1.0 + } + } + }, + "versions": { + "arc_sk": 2.0, + "hellaswag_sk": 1.0, + "m_mmlu_sk": 0.0, + "sklegal": 1, + "skquad": 3, + "truthfulqa_sk_mc1": 1.0, + "truthfulqa_sk_mc2": 1.0 + }, + "n-shot": { + "arc_sk": 0, + "hellaswag_sk": 0, + "m_mmlu_sk": 0, + "sklegal": 0, + "skquad": 0, + "truthfulqa_sk_mc1": 0, + "truthfulqa_sk_mc2": 0 + }, + "higher_is_better": { + "arc_sk": { + "acc": true, + "acc_norm": true + }, + "hellaswag_sk": { + "acc": true, + "acc_norm": true + }, + "m_mmlu_sk": { + "acc": true + }, + "sklegal": { + "acc": true, + "acc_norm": true + }, + "skquad": { + "exact": true, + "f1": true, + "HasAns_exact": true, + "HasAns_f1": true, + "NoAns_exact": true, + "NoAns_f1": true, + "best_exact": true, + "best_f1": true + }, + "truthfulqa_sk_mc1": { + "acc": true + }, + "truthfulqa_sk_mc2": { + "acc": true + } + }, + "n-samples": { + "truthfulqa_sk_mc2": { + "original": 778, + "effective": 778 + }, + "truthfulqa_sk_mc1": { + "original": 778, + "effective": 778 + }, + "skquad": { + "original": 9583, + "effective": 9583 + }, + "sklegal": { + "original": 1334, + "effective": 1334 + }, + "m_mmlu_sk": { + "original": 13062, + "effective": 13062 + }, + "hellaswag_sk": { + "original": 9485, + "effective": 9485 + }, + "arc_sk": { + "original": 1169, + "effective": 1169 + } + }, + "config": { + "model": "hf", + "model_args": "pretrained=slovak-nlp/mistral-sk-7b,peft=/home/schwarc/diplomovka/aya_streaming_lora_unsloth_2gpu_dispatchfalse_225k,dtype=float16", + "model_num_parameters": 7283675136, + "model_dtype": "torch.float16", + "model_revision": "main", + "model_sha": "089497ae0d72018e9591895c54f774bf0b4a83a4", + "peft_sha": "", + "batch_size": "1", + "batch_sizes": [], + "device": "cuda:0", + "use_cache": null, + "limit": null, + "bootstrap_iters": 100000, + "gen_kwargs": null, + "random_seed": 0, + "numpy_seed": 1234, + "torch_seed": 1234, + "fewshot_seed": 1234 + }, + "git_hash": "53504a96", + "date": 1790780827.9713795, + "pretty_env_info": "PyTorch version: 2.6.0+cu124\nIs debug build: False\nCUDA used to build PyTorch: 12.4\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 20.04.6 LTS (x86_64)\nGCC version: (Ubuntu 9.4.0-1ubuntu1~20.04.2) 9.4.0\nClang version: Could not collect\nCMake version: version 3.16.3\nLibc version: glibc-2.31\n\nPython version: 3.11.16 (main, Aug 27 2026, 14:44:21) [GCC 14.3.0] (64-bit runtime)\nPython platform: Linux-5.15.0-119-generic-x86_64-with-glibc2.31\nIs CUDA available: True\nCUDA runtime version: 10.1.243\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA TITAN RTX\nGPU 1: NVIDIA TITAN RTX\n\nNvidia driver version: 575.57.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_adv.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_precompiled.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_engines_runtime_compiled.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_graph.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_heuristic.so.9.10.2\n/usr/lib/x86_64-linux-gnu/libcudnn_ops.so.9.10.2\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 20\nOn-line CPU(s) list: 0-19\nThread(s) per core: 2\nCore(s) per socket: 10\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Core(TM) i9-9820X CPU @ 3.30GHz\nStepping: 4\nCPU MHz: 1703.826\nCPU max MHz: 4200,0000\nCPU min MHz: 1200,0000\nBogoMIPS: 6599.98\nL1d cache: 320 KiB\nL1i cache: 320 KiB\nL2 cache: 10 MiB\nL3 cache: 16,5 MiB\nNUMA node0 CPU(s): 0-19\nVulnerability Gather data sampling: Mitigation; Microcode\nVulnerability Itlb multihit: KVM: Mitigation: VMX unsupported\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Mitigation; Clear CPU buffers; SMT vulnerable\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Mmio stale data: Mitigation; Clear CPU buffers; SMT vulnerable\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; IBRS\nVulnerability Spec rstack overflow: Not affected\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; IBRS; IBPB conditional; STIBP conditional; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Mitigation; Clear CPU buffers; SMT vulnerable\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf pni pclmulqdq dtes64 monitor ds_cpl est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cdp_l3 invpcid_single pti ssbd mba ibrs ibpb stibp fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm cqm mpx rdt_a avx512f avx512dq rdseed adx smap clflushopt clwb intel_pt avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req md_clear flush_l1d arch_capabilities\n\nVersions of relevant libraries:\n[pip3] numpy==2.4.6\n[pip3] nvidia-cublas-cu12==12.4.5.8\n[pip3] nvidia-cuda-cupti-cu12==12.4.127\n[pip3] nvidia-cuda-nvrtc-cu12==12.4.127\n[pip3] nvidia-cuda-runtime-cu12==12.4.127\n[pip3] nvidia-cudnn-cu12==9.1.0.70\n[pip3] nvidia-cufft-cu12==11.2.1.3\n[pip3] nvidia-curand-cu12==10.3.5.147\n[pip3] nvidia-cusolver-cu12==11.6.1.9\n[pip3] nvidia-cusparse-cu12==12.3.1.170\n[pip3] nvidia-cusparselt-cu12==0.6.2\n[pip3] nvidia-nccl-cu12==2.21.5\n[pip3] nvidia-nvjitlink-cu12==12.4.127\n[pip3] nvidia-nvtx-cu12==12.4.127\n[pip3] torch==2.6.0+cu124\n[pip3] triton==3.2.0\n[conda] numpy 2.4.6 pypi_0 pypi\n[conda] nvidia-cublas-cu12 12.4.5.8 pypi_0 pypi\n[conda] nvidia-cuda-cupti-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-nvrtc-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cuda-runtime-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-cudnn-cu12 9.1.0.70 pypi_0 pypi\n[conda] nvidia-cufft-cu12 11.2.1.3 pypi_0 pypi\n[conda] nvidia-curand-cu12 10.3.5.147 pypi_0 pypi\n[conda] nvidia-cusolver-cu12 11.6.1.9 pypi_0 pypi\n[conda] nvidia-cusparse-cu12 12.3.1.170 pypi_0 pypi\n[conda] nvidia-cusparselt-cu12 0.6.2 pypi_0 pypi\n[conda] nvidia-nccl-cu12 2.21.5 pypi_0 pypi\n[conda] nvidia-nvjitlink-cu12 12.4.127 pypi_0 pypi\n[conda] nvidia-nvtx-cu12 12.4.127 pypi_0 pypi\n[conda] torch 2.6.0+cu124 pypi_0 pypi\n[conda] triton 3.2.0 pypi_0 pypi", + "transformers_version": "4.48.2", + "upper_git_hash": null, + "tokenizer_pad_token": [ + "", + "2" + ], + "tokenizer_eos_token": [ + "", + "2" + ], + "tokenizer_bos_token": [ + "", + "1" + ], + "eot_token_id": 2, + "max_length": 32768, + "task_hashes": {}, + "model_source": "hf", + "model_name": "/home/schwarc/diplomovka/aya_streaming_lora_unsloth_2gpu_dispatchfalse_225k", + "model_name_sanitized": "__home__schwarc__diplomovka__aya_streaming_lora_unsloth_2gpu_dispatchfalse_225k", + "system_instruction": null, + "system_instruction_sha": null, + "fewshot_as_multiturn": false, + "chat_template": null, + "chat_template_sha": null, + "start_time": 6683425.455494504, + "end_time": 6714088.051436605, + "total_evaluation_time_seconds": "30662.59594210051" +} \ No newline at end of file