From 2fcdc9edbb24be27404cb2b796ba6e57b804c899 Mon Sep 17 00:00:00 2001
From: Dominik Nagy <dominik.nagy.2@student.tuke.sk>
Date: Tue, 11 Jan 2022 23:01:48 +0000
Subject: [PATCH] Upload files to ''

---
 BPETokenizer.py              |  19 ++++++++
 BPETokenizerTrainer.py       |  13 ++++++
 Trenovania.ods               | Bin 0 -> 10650 bytes
 WordPieceTokenizer.py        |  21 +++++++++
 WordPieceTokenizerTrainer.py |  13 ++++++
 all-in-one.sh                |  24 ++++++++++
 idoc_install.txt             |  13 ++++++
 pre-process.sh               |  86 +++++++++++++++++++++++++++++++++++
 8 files changed, 189 insertions(+)
 create mode 100644 BPETokenizer.py
 create mode 100644 BPETokenizerTrainer.py
 create mode 100644 Trenovania.ods
 create mode 100644 WordPieceTokenizer.py
 create mode 100644 WordPieceTokenizerTrainer.py
 create mode 100644 all-in-one.sh
 create mode 100644 idoc_install.txt
 create mode 100644 pre-process.sh

diff --git a/BPETokenizer.py b/BPETokenizer.py
new file mode 100644
index 0000000..9db172b
--- /dev/null
+++ b/BPETokenizer.py
@@ -0,0 +1,19 @@
+from tokenizers import Tokenizer
+from tokenizers.models import BPE
+
+tokenizer = Tokenizer.from_file("data/tokenizer-wiki.json")
+tokenizedLine = ""
+fileName = "eceuropa.test.raw"
+
+def listToString(s):
+    str1 = " "
+    return (str1.join(s))
+
+with open('raw/'+fileName) as read_file:
+    for line in read_file:
+        tokenizedLine = tokenizer.encode(line.rstrip())
+        with open('tokenized/bpe-tok_'+fileName, 'a') as input_file:
+            stringified = listToString(tokenizedLine.tokens)
+            print(stringified)
+            input_file.write(stringified)
+            input_file.write("\n")
\ No newline at end of file
diff --git a/BPETokenizerTrainer.py b/BPETokenizerTrainer.py
new file mode 100644
index 0000000..1f4de60
--- /dev/null
+++ b/BPETokenizerTrainer.py
@@ -0,0 +1,13 @@
+from tokenizers import Tokenizer
+from tokenizers.models import BPE
+from tokenizers.trainers import BpeTrainer
+from tokenizers.pre_tokenizers import Whitespace
+
+# training the tokenizer
+tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
+trainer = BpeTrainer(special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
+tokenizer.pre_tokenizer = Whitespace()
+files = [f"data/wikitext-103-raw/wiki.{split}.raw" for split in ["test", "train", "valid"]]
+tokenizer.train(files, trainer)
+tokenizer.save("data/bpe-tokenizer-wiki.json")
+
diff --git a/Trenovania.ods b/Trenovania.ods
new file mode 100644
index 0000000000000000000000000000000000000000..23dbe079a68d941318f6537af79c5019ba043545
GIT binary patch
literal 10650
zcmdUVWmsI<(r)9D#$AHDy9IX-PVhkE(hW2YL4qX&4GuwrySpX06WrZ3K?4N1WX?Ay
zGxN=yd(Y2Xu%EqlKfB&lYcH*;S4{~H9v1*W1^~k9n6-lK1;f|?0Km`d@h5<Tje~`Y
zr=x|bqochI$kfFK0%msuo3lYop*B!9h@%DA90GE6umHQTK^>hfOwFOz78WjQf5Lo(
z`L_^0mZTtHOB*X!=iktvT<k6oh`pJqGy8w1WoZvFb+PzwwH_^X{m*p~|BZ!?5Jy+X
zpQ``CM$*60GX;Sx>@6M}3i(fV{#}hfzlozM*uwt5tkKaKV&!ZBh5q+#gu0lzxc*=C
z$p5B|HV&p%7EpFc8y5#tN9g~Ui;Rr?+i-e(|K9?9EIF8hZ7eOIE^N-`mcy~b;6M({
zpcCO>i;-Dzb_4`xhJi-gj1JzA7YOX9DcH+xk+C}JIYs*!lN)8S+$)hasYj-yeebFY
zK56lQ7-<LOt!wv~g__wKa~Fi>XX}{BQ=Wda4b078YH|lN37RAXv+@J&N2{|LJW|et
zpSMtbni6o52lmuNBBPK+NZMR%`Vb3_tfN@!s=FLyMtfdSh`d`8D$J}xHLcTV4XvYy
z2@sD!Q630#2_$V&RqEzWI8zt=ly$V;`?7T+FkATLSLVpk+2uFDye;F^){0}pz!jf(
zYp-LFm+M6Lw-xHRmmj!}pYJz6^L*I3XB&MWS}(70leQNh`Dk?$y;E~at}R!tri6rC
zb7eh_00RI7!T|vPngf6L`F@*?7A~f29uD>qu}V>?9Js+(cX)kitlzOHK2e2hX=@W`
z)@o=LHouv5FEHqh3?~}*Gcg`f7SbC+H)(^8ip;1jXG95k!Ok$*)xDvEmw6c!Kfq&f
z4!UAo&b-|>vLPpr2e6gyM!xj%fj5y0oUCba(%5Ic*}8iRTD*ESUXFc0JcO*bhB!Ds
z^DXxEy@)>UM~`<xRt=>FQ_h;?SzE@A-mo|<-?<0#E4R<|n9p_+NLa&QpF`l7x`Us&
zsy>gyCFBXeGf>6}gR?7b?byJDE@M)6pIu8n&##w#;l>~QBHe`LLl%?0`;dOTrrBEN
zB^(yULSPtHkC`Os{Soq7&}lf0eJuU~%b@&Q2!kTy_xQN+rv~4`on&KQ<JzU{bmw)!
zq#8*ui5dE^JmbG8YWot^{DXV2-TeG^{Vc38A*@|e${_dQi;|}$2?xiU(qgj7PoHKO
zMMh$I^ym+dC6bR@9vb_vJc(NYm-JSCfnNn^-N}V=Er#%8cYL7_gSzuKrd`v_SSqe2
z<UC!U-Fh%m<)G^R(^)z}*&^YpS<0AYiaxFi-={Iu@8Ev+EE?sEZ_Z=Ss{Y5GMSYwY
zP!~^o3+T_jUD7vzEb!s_u9RbHy`1&$kK4?NsO|bR&Dvc1X{=@zB%u_i9EXkYJ~H<H
zCI*-GfULVP4Xik(wdsyNdOae8dvrF~9QUIBjgxZqlZDjAy5Op4Spg-sLyVL158SuT
z*TaGlAku7#>@_->WXFaE`Zop|0vi$|n2CXE)=x<i5OAG`0)csNkf?yxkyRQzy94lE
zb7sPuK5T4<wV0lO4!7nshaK(#?63Dv)u55$?yy?Byyj;6?}-gf-mTDSTF_3S*rX_V
z>%vdPO9R3Aw(2hw?2SHbJ%hwcOVnJ>x-V&VsHrA1d>Obde8*_tX@FPNf6wG-BqRf?
zil;N%bolhsTz*@U*Sl`m-8s}Id`6;>qB(5hiNec8r}Zuu3SXY|qL3bqk@9si9m{PJ
z*_s%2WK0RgAUMdjwoOU}!~TV0hzEKODehQfxAkpyRoHk;jQ~vd9UIclmvhn&auKfs
zX%;Q;l!}O>_UPV)-LRZzHIs6d<WYPvlaP{XW{@@dYM9BP&B95Qi-cDL(JP!(Xbj{I
zARG=%apBHW^z|~lu;ulH(Vb9|2|O5|qcfu96C~rhOtXyzgXK2bEnaY`1%`ff#LEtW
zZd}&_h_w_ko8(>->evwzAHj!Rh$m+R)H`cDoLvr=&FTros|P#qv_-G^I4`h_MVRO9
zi6421Q<1t9qb}El_tu6MLf8lCt(3PkxkD(Osc^B|gxSrE(G0p<s}4W=23o+{SeB9`
zDNLPy)0EUyK4w3Jn5te!u^Mt(8=y_0p3pGF#>5DIu1`8cr0S)L%W4P)9wY@iO_UKz
z!tqVz$pRa8U~`{WuBM<CE!7%7TqnAoc}1u3IiiZhmcU7I@@{l*7o1Z`axP+4TQKpG
zNVz5pQc`uXtqGcQm~xmrYdmy(gTiv9*BvLV#LDANHH0vv6$-V=7g*rTOTRV}bl4HM
z?B=21P~zi=sb-jf%VM$rl;cSbKCDBP!^a%*`j{DzTZMh)X*5Duy47`XPzJsTqT4M?
zRg9lTt?Y4Voth+{5t0`<WFLLEY_IP^O+(jc*o;s$WE94IH>;3i7=**pP~zCC7B#{}
zwuIozcvu;V0|R{W5Sclj6oqV{jji46lHd4Mt!TFahbhB=-SE|()>!4*_iC-xjLmF5
zokKeAIXlWZGzgaJbk$55!&_T*4}aW^8M&3a-FUV>(ovKacVgthg~6-$^{ub2M#$Eh
ziB@{O{T96+l3T7YcJI7xhGVh4a*WMmCB65iWwm{@v00~eIDW)&)IM=mgavs9ud#)3
zWD(JfgbBQ61anO$<2W~^XzDY*o4ovz*7_dL%1P1Dyj1`?HR_mOj^I>Qu8O$?hxjc_
zBjy!RRp=cTuJX4hgHIXI+r@2`m7axNO~47|*`jC3A(Z54=4I4j#S^A+!bP)2PWF(1
z;h%~FiE7z{`C-G*#I%ARWNm5^!JuhTUb2v;p8UA%gEpVJiRT!4VKdg$vurYvGGR78
zn(}%qk*WiuUw8Ggca!Ah2`@!mF%<g5qAhNZX%=#Pv?K2Z2rYU~kmpAkD<dy1XcWon
z>+czXx%s1K1CEaMvde_19qH`AkeXM{pE#Uy3_a{U53NZ&rZ{SjuJXRY&y!WQztWlC
z{#Y{{rDBj%z8Bdd<@4iwZFVVhv5?$K%7EXt<-~tlFTGLwAV+O=jFDck4STcCC6naK
z>a9b%mm5Zbw|u_p`4-Ncf(^|UGPKK97tQV(bw$*G3smWI@!l^bKrK|;ZX@H_89_eO
zkn|`wn$|fo^TjX^Qt~H$IbAJeSHSVeV)vYa(=2jGDc81eXzuInR!cq=*t8eQZ?Yc5
z-kIKg_>_^x;<f{<W$HK8@<9_1Yb<ZkD6zWTD9Q-SCWg&_U#9xmW$_MTJ~^W}-+7IO
z*qoRwPg$;|B{B?TQUS@$UWD!1cm-b%LhoFI7JqEn1)-jwJ_>0K(y;t8RTiEvUjO#Q
z_s@}(YY|b=xcpQ5e#|CA5nRmz)h1qBUbU5xD7i(5dmrii(W5ZbZ%2F<KtX!#o%T+>
zYOif#mZE5z2?2vAVGk%Q@=x^_{Z=S*D$PLhx|)Mw$YJBUcdnr~)9d<Re&%TClYZG(
z%d0nS8(xw0(0<o|dsib+Ff5y2g<{}Ds5Y}&;e0C6vBGLwO8>T_v-8dAkggK^O}%py
zJ6h_!hzU3W3(ghhPGZ)%vfv3W2x-7#Y(+%;A+bB*0rux^Cg8?WlYj^S*wg)|bIOy)
z-3$Z)yZm$s9=Ee2!zJ+JcGk9{iA+}aOh4hr2`ytC4?-QC&6k?lV_5xc^RhUGXsyFT
z+mj0Z#n|x442#~=0y?fk9eyI|bm;dqa$n)N{j^qngf0*9a1e32dkMH4lsT5ghJJT8
zk{<=>$T{0zZ|2_joR#2C6ms&HgdrtHy*81u4fVn2?ApS?eH)V$%#A9zU5$zOW;BTV
zBNi%jX-+duuL>-5?6^#@lNOBmWr!zn(87K0L}NRZTr9~J9k~YElI@6lhPFB9$haJG
zllKGx9)pmzeUw+{3D)6cMczbV@(_MDscur;kQlh^<O2dKMF%AThl|=;2P{_>Nw9Ro
zp{U<iSJ&KvAt}s$%n=kO5+@?VeSP&Rt~m=={8F4up7G6{HVl;zssU-X($lQwr1wi-
zb%e2Vj>@sw?2zQejVx#lrj4*jeaqP3cvIhbX}QPX(RX~rQ^-m$BidIGX_n8^><BlL
z#f~lmQN4~s<u5JMwntLjOCLfTI`=f9d!^g1d;^{%iK==T&q(mZ0)LE263A>JPeW4(
zC^+bkpu+t4#opm1Xhmwhg!%0?!i)zG`cfR7TavDAHQJK}5>tqBNqV6<YB>j0S-LJR
zyw3t{Zfml0Zh6L~dF}f)f9Ty&402*^r$7>QUfRq4xaV$3F3`sM%mm;da$Hwh>pGJR
z$;c7=Y+*9;7IBcTM1Z+EtKMuTCIdbm3kN4;)BJR2yUFmf32Uj$@tA|uM$>|+Regx3
zKsa)eObR0m1NF7$;2A2yf^z;0ecJ5<y-{zUnVv1d;fTyKNsfP`9?+vo4nXFD=@=vR
zy<1P;xVs`ow3y<9ds$`46wTY7r$AZ`&!(+x7`mvPG;&kv>*of}doO!uIZ8JJq9pa;
zt4Oc0p>32{rfYpAV&WmOr(9Q3-kq359a<^#aH}y-aD|aRvEuPK%{2TtDs^(#BT93S
zNc(D?#a-;264CiWtue6z4*Qu*Rct#+o4_O!{$K@ts$qHx4f<!*GCeZ*7#h7Uc-h6T
zyY?fPKY-iv2^BmA2_vkQ>q>`34oe5Bbeg1e-<2FMO{+12hxgHs)(XfP?$<izI=@wr
zHGEp@h{yvqlQpbmb<8ap8`*#CbO_zEHk*Ru7d6!Bb%o;>G+gc3FjaA!=nNp4UM$o(
z@t^IPZlG%le2U@bK1q(Tp9#{oBKSDSYVMo^WKPovd=hjQtX5gsgVXrkw~JHGUltZA
z+P|vqCFgiA>neEh`X$9Pzb<k3SMVAUGkt-aSO%)`q>2y{wr#9{D5CVClqQ_jF(eq*
zT-wk6Niz%pt$06ac~KmRJYz{(3~hglkL}t73MW-VumbsAOV(cFC=|EOggVcQi$x@g
zl`}(hwIgIy%29LkyW>58D3?+lHbaxG6P?A20#aD7yS<+YBA~Eeq&Yk6auc%Fnbq-m
zEyw7_<><rp`H;fC7S`@*zKqhF!?VPJ?u(YOqsOrLC9-2;6Ij_86?&T~zvm-&q|p-G
zqOGsnTN^`wAHiB{6iW`D=LKGm!@`|%^<nB>#T#iTDH|ejC6?D~2_^p+hMzE8Wc1f5
zGaPw8Z%pDv#D0XS(zIfKlGCVC-N?L`SEteplX{0}k4{`6Xy3720F%0bXs`PC;Lwq`
zFJQd9OK?Kt8GT$)fPPilo!e@pR7}m}hX<<=?(Y%4DL|lns@2=KB6XY;dpV`-;-sL5
zd&bS}EzDg%BSQGzy{NgvkJ9qT?Ufjhp8%ry404rPBq4dGIO<bD)E9{}?QQMtuddc)
zC7h8X#pU%Jfqlk<A5(iidyMXqj&OcjS2uN;SObY8ivx7U$3_{1!`9nlSeZC|Ek|IV
zY`#gNbw%7d<IHRh$s!+2|3O$`D043G`gHJOD*qEFpfN|{6b8(m5JBmAEWAPCMQ4cn
zzz9L0q`5G=8q${974sH}Qe)to$SoVabmR}xY7x(bPksH;hN^=2T>uz41%_ky!b=}H
zZdKx7B$i+))VpO<da-$zEEvh?ZXbWK)q!p^!LqBA0QtKrK&tuK7<0396+ZG=1G46&
zN_4Hy-02qo=R{VyHm!7eBk$})YuY(y^T8NaIWiGB><fflyCY^fuXr)rg5%QTmzlkD
z8eikCt6GEVgRV<4w-tEe_rz}?xz(d8q&X&oG2Y9S89p&yFi8*!Iz)7|9OEM51ejpX
z-McZ7c>cGl^Ov=x)6TQ?pG-|aomP9xF&Ph~-M*Lg3A*%)r?vKTRThH@p)c-^SsR+}
z6ipcC?~bjD5GZ4WP223t`&douf;{(w1gDg)YqG+lyq40J36i8wgZ8Q%*bVCJ=T0}-
zH_&PmpW77d3$>+odoNWkyUEjFq(rVTLRUcje!M2B33AR_Og!N0e97F57RV1_Msj7p
zjyN%g2S^Ot{cql(OEHW!0TKY9tMH%p)1P4zsD+D*4cO{8|L{oPz&%?C*H^4PdvK;&
zi0Wwe%aqWQ(!}KwSmOZ$|AOR@Acfa`KW=c;tn`sANrTHq^z$+B<WG;%(%5gY8-BE$
ze(lHRiezYH(iUaN#gQl7r{Zd7iWnN%%OBtA5leL8lR__Po<tFr?_vy<>|^^OjZWWk
zti~nfEZ!lX1{>*-2EaINa@?N`HGfueU*#@>Rp(pv`HiqiJig@j3ui%KC&t<w(!lnX
zVyO-@ed$73-1x~;&|$5mf|a)V$l$^4No#ELE4+mvE_Hk8-De)#!phoCx0{H;VZ&*$
ze0g#9y5at9Czit^8tpQ6EMU7qX&m42MH1(FUym9HGij3Dfkb9=P*z&HFv~E(5Am9M
z)ms3ko8oyiGrrB36@e8&Mr6o|1dmBhk#CvS=vA=Tg}-0en<u(sOLz#K=Q4^t@}DcL
z$%9QjHjb_P5n_Q1yk??W_wo~jE^^Ot<O6@GewmR9M+nNULV_`X*@#19kdr+1OR+qJ
zBRJ(f)g*Sl;MgRy*d?Kbec9P7FRVszoZ=@jhXo8(rpRYFe|l7pH~<Z8n`{U83}acN
z;J7@E_6+fy?Chw&**>hTtME!t62#^=Icb$~(4Z_xq^^)Dv{H;SBzhVc34`vYjw8do
zpT!4&65dXmWb0zU^W*aza@8EvIVf_%=S*U<_#=VQ*X2QyGeOMX7$(Aq^43H?z|jx9
z=Y>nMLRRwp&c$QFOV4S(^6G<hV))lubZWU<!BF*ZhYqf@V8x4#0QGG@$p;D-mQ2R8
zyd9W8r*$X5+9ei9LKf%F^S+tB=R{K-*M-wSRMtUVSx?BQ$g1e(TZh)pR2BZqII>lZ
zvkqz@G;hp?Dj<`z{JMNjJ|o~RL<~_fA#&`EaH<)&W9fVPyXc^ABFQ4F0)f%Xl`3Yv
z$qM<@d~<00$X`{8;Mi8#3QsXB5EG#KGy1^_W0Z-T11u~|``rP{U;@$FCIS3qWP1Wu
ztWhm7`mZiq7`Y?F#s-vNX)Zj|xJ<|#%U$!?2Nms%<p8P5hsNO>i4O6hJTbuz8*je2
zk(Dz}!+v3;>uhRhI_#n$mHY;B^m_NA5_d~LlHBW$SfR)47Mr_4&X?E_Y!B26d;~f+
z$om(#Nv!y()j)GkJa%T>Psf!3XaXha{8%t7OuX{5xAi3rvzI+sNy0;Ntp+P}LT=mh
z%|A#O!M$B7pdYtoI4a&_pUJNrR165el`}sO8KdnjMV^{l7vtLuH2P_k?5U{e_6G2_
zUSwo%k9~7ob?a3hEbhOoY&m9lsqqAHV|$#^>zwD4;@|;$8e7i@zXhJj!qU{|qe$C?
znYBovrfT}MJ`JaruFp)CY)mRaAM9vb+NZxuF#~5nwlPyrL&5IG-ru=epkA-QDy;T{
zz8dvbgA<hpYwQV)@iFJRpK_v*^4$3Mc^0H<@ejPxhirE1muxi6hKloapOlzFzw<bK
zlke6ZJWsuwMLyfEEz}C>tQ4JiwvjbQCZ-ZUAP$K_)Hf&>Si%cA8W`V?EQ%WC9ZfMY
zY8_Nop`(N?3Evc!2(3Qnx?69}tCUG>PC22cj5dOUS*GyVOzv$nD)M|DP&`N|Y0;+E
zL1h<^$?Bg5fQwPcrq)c$)*yvr$!bIvP6Y%#<Z;KUz#k5#hFABp(L_`3m`|KBaKf!h
zO`A?CMKlPONZG*<I_Bgw;J%FRO=Z0T^?OUF%e67<rQB)kAHGWG?iPMpzEUan9zU-I
z8DjLku^+8HFq}qamVzP@S$rV1Rv5liSCFm*#llAKjY-qmOzJ2VP+?gTy^1HCwsf7<
zL+J>g0$sR6&SLs<E2C|k!Rl4}_;-w-E{{p>>?<1t004&i_a*xGL`kykXA9-W$lcHD
zXHrAU+SS1fY-(c<Wq0|#$mR&PdaI@?hlWc0GscIeATO=)c<+3?9*`eHf&g!#_o|OK
zOEqOJna4X22*km`AtolKrlw|MV&dlJe)jB{w6wI6l9HyTrh$P02n4dTvx7pR-rnAU
zfr0Pdy^D>FO-)Vx`0-<LadCBZb#rrbXJ_Z&;NZl>#KOYD#>U3M!NJ+t+5P?f&q^O2
zde2zH005LM1!)N_kNKn2x8Q|7Y#C~G?E2-a)`tgnlb9;2_b6!|bRL=5xG==;8>jY;
z+>Gkl&R6_yPoK<K-d$2Alc&)gEFgUmbzPwIbL$aHzsl{sz1tQeZ)jM`s}i#<{C4C=
z`^rc3!v&d_|GRt77wju0>+PaPuxk#ZHfJjj+=F6-WKD7($uJy_Cxjy2(cGmZda{I4
zc4xUM=50w_TM#V-1MMp;`zOTL1r)A5yyJ+;2bL3>8@?LoCz25D`1YT@Iy~S?zWq*?
zJO4TB44y#K&u`4;!#=M`Kj!(X)=zqgi@LB#*D~ybi_3(k9NXoDWnXjA1ko<Pye-g`
zy^V$pkr5Opd)0504AL%(aKwF)N8Nf4Nsc_EtB)4gv6Q!Q1Z8ZE`AZ~K8ZYEdd&?8}
z=3hbfvkYgYxP*eTpyOqG`fTxGq3`NT$;dWD9EEbcTb!*qu#QI(eM(JcIvJXGq{0Hz
z+!5QRl_D>lfH8pNF})q7CYa7z6;-U}nlEm$P&F4jZpys75KzXXT7bktqXV$mpCqmV
zuS@qLx?He=*02-qYnDLy)_en@1L<mz9S=rj5an_`vt2o-bnWH>zXRj*dbzO;>!tC2
z;c*x0`kEn4(K6&$?d-0`WZBCJjpz=SDf3fC^zSoSkAU8<9Qa#UK}2&RI2#FkHI#;#
zGBGxFe%!kUccydj!|%ktPX=4p3&)XgBt4sh4dzO<60ioNmigEz*|pV2_0VRE&Qx06
zCRZ<>GgIfqJKwN_ilZ`G{O%`b?finXY>XZrZoYaEyHkiC@q?_9@Y>tV`?M7q7OBi@
zKjiN;&}i{4;V;C?=V|B1b7s>~-Ufu=Q`^U9Yncu!09g=WxRVay3y<_Ll=vuySu`Bx
zB`GdB%fBck=86=A)?Cttbi6>dni?M`ncFe--P+W{_^3ffh#l;u7Z4DrT*Hu@z5Pbw
zT-!-zRo05LWK{0SlRMstG}8#aA)Bztw>4blT_Y6R%3DZ4n*De|D1IQ1{_Th>6AMEv
zH_wK51<ae*g;QO~Cf+LjxZWIT!Hu>Ibj>AjBE_0ghHZkSQoQ%6P73$UY{ZNa<y>la
zdc;>qT+(+{oCfuFRR<CWKbG?ib}PM1lnXvdBe&;dM&(!BqD<FVf&@?`W?A{xa;P|K
z(*kI6`6Ea+Qi90N-dh;cxc4S|>YtL95Kv6e?{}6t^^|-_v=U`)nB$4m8izAY^JbDG
z`s^e^v#i@IAfmppOy*!d%38H!__3#<M(|4=535zQsS#fY#y$)Wm^E_jY?~NpDcNYZ
z)Ks9_GJ2g7lWC2aF;ZNkLddKr3O??3kCl#Ac;P(XqRqQRv%@E`yAq|8>~|`g&UIA;
zI>?_$zZ60lCogrPy}t)awA%IsU^18}=_I*VoqKaR5*kSCRmE?%lF3TU<vB4kgszC`
z@hT8(g%a!5WZE&)+ZpD{5a+pvH1exi6Wc5iT3adBT!Ye<&l=!?7c7Y5OgSdO)9I@x
zBbkJ6-z8BWy^iG=;4L%L^UaxvwtJq^S4}8<j<js~nT03x`x9aUvrTp(wyDuJEL+7|
z0*emAbNdj>NZCr=#q?p1)K($3bYk5ca9#6y6V_baeng$>B-J^VRP8t~tzG;`dKcgt
z&K!*@pl9l)B(#=Lfxmoww{C0+Vw9~}C-1&_#){Un)b}_@cCrUpKM0EigvNod!ui`>
z5km5H`w_1DAiT;g;%rPR<$l+t{lSgOG5X26#22Yn!n$7P_ueym<~sO`dce;_cugF+
zr#V|AYPCnk-W1MFxOk6|Rf)*bDkr&Rpv%*E<>jH%*qfSw<0pI7=Q4AU9@^^@;Yx99
zkN5yoUUZ`Svo2RiH`}@O(Zr4&+oc)vA#Ea=^H6_$oZ-=Z=JWd9Q@6}UADz#vwXfR-
zBgVQyo4Cg)E{LsZW8_zKRYVltt)_c`+D0kINv$9yn6eqAPSYil!bD^rxW|q;PzH=l
z!mIPRv2WG|^Fwb|O+G_vvwQKEe@rL9+A!g$p=eqj!3FiJ7)<X+5>2n841eQk=S5I&
z3V54Q5^w^1J80Q>l8`!3ds%3=ma`dM;CXM9u4zBxDBd;9B4$-A$uk9S#pnD*lq_*`
zREUL#=ZtA%5(y~guBmjyKH^Ut3KUzQ%8CvUYNA6doE>>7IQ*f11cAQ|M~W54(5Z-8
zBq?QETTNgUF2<_VmBImfCA|3<)U^@`4=Ziy6)1iyIA*h<y<&XI;vIRDCR)*E`O)|@
zWLT2BntQ0$U!%kLT}WSTeG`P2V*)dZH}YCu_1s{FzrFVS^tfJegbJ&s{C$WFtW>Zi
zRiQr`_Bb`7wQa0f{&gw3;)|MKl-_#5FQSlF6=Wgs%_p%m^X~jKh2UDC4VxU$^9{5u
z<I=c>U2XN~+t!mp_sc*h#<&cWwm5cb=K>7&9>>l4w(Xngyn3}%uHh%w=LW@9!>%Sq
z^+gTFfxIKdI&p$#1;*E9+71I5wW|KN4k{Q?`tLO|peMD&wchbMQ#&`)4PM2{F~@y7
z2K%Hwn?HmzP8=MpXTWPUEj!tl<rk#~w>LU!p%;!3Y$wGcA_PkzuDKE$<C+@@*i^X3
z)nwLu>%$O-L+W-{)`YP$ABZ(<bhd=Mb<6C|d<GToIeelGf0g;#-L_={rnv?ZgwS@E
zZ?NFhG9hC!KMT=*!m!~7&x?mkgNLOi!1FhL|1%PyRP_%mTsi42-ZwrAiTfmnwUcB@
zBi*{Q24ec3=DC!S>fzSp#43KYgB4LRZPBkWobC%TCQ+I_G!$NJCG=vohsbcqwT#R8
z7H6#aM=Pcs)Yl!KGi_?-lfVA51lbDm?%Qx{$2)DHCA}%lbXo{FyVDspC`cEBs$;^9
zUtHcIISkrA4o%T=UZ2f+M9D!LU*_SrGi}4k=eNX5eKNd@Zk?_c`d-oErn#$>j=gPz
z(7Fs!PXWwX8(+J%mVq9k>hy2(exD$C>lXOiejRFl`}N!7CLdrPXIV7N|Ihzx80O!}
zLEFfFIxU1)kH<EEf{dzkndHmY|4a_RdwfepMoWTKK~<Lh-z1C2@Ox>j3b>O3Q}#+Q
zxVw(4Mo*hT6+g!7^*LY?h>lH$2~>B!HDgva)Q&M@eC8W+C4yKvm*%aTMnQK_X-fVh
zP20b82ERGv6?xnK^SXsHzE(*)ikJ_G+Jd2bd=4)n8Cn(j6tqNW6iUxw)Nc8%{AnJh
z2RZlnQc+6g@Q#E((1Q;*;kiUV>Sz^}OE{uq9#!OiIPaOo-wfxVbI_f-L+ueNQ1Wb9
zL{N4sLl28Y!@+1hRslF@t0V%@G${oZA|=;@Qldmw&x+4*sTk5C0bO0#@vU;8W>PsX
z{de?u8nVVh9o1V$A1YXU^f%N~>0c{N8K4*?!Q>!LA^9+!(Bn(t7?I{Af|QtTJcbhw
zVAZc3$_YeCp$~xrO*GcTr~XTUm<z?|T@@x8eCsPwEBC)8I+B)`)i@v1_AKx}PyH~k
zxPX5qNd4@eU-CO$>aSYA-uRcd@atLYvG~t~s6W^JOY5&Fk7xS-h4PCa_5X^}@i!>H
z2vh%z@{_>yOD-Rae{Inp2~_`#^OHjLOV}QJ;IBA;CRF`1(w{A|{u`t}60H6i=g$^p
z{teEb30MD&^k<9a{|4!g1gw9?`ANb0CD^e4`y2f)Le{^)e&-bZk!|!dw)jh)KZ5;x
z?RX?;{XU3)*Zd=!<tH2Imqb1~vyVK%-?&MC)%s)B@TV92OROIq>tDQ=zv}%lI{GuG
z_)GX6&Hh7%@vn-1^x6O1z!022hf;sn{3oCOSCx!J|Lo!aj`I6w`s3>Hb6x%=#g9>h
bzg?x(ln@^CssI4i<DbRjqC-ym^XdNp#CQcA

literal 0
HcmV?d00001

diff --git a/WordPieceTokenizer.py b/WordPieceTokenizer.py
new file mode 100644
index 0000000..7a9dc14
--- /dev/null
+++ b/WordPieceTokenizer.py
@@ -0,0 +1,21 @@
+from tokenizers import Tokenizer
+from tokenizers.models import WordPiece
+
+tokenizer = Tokenizer.from_file("wordpiece-tokenizer-eujournal-sk.json")
+tokenizedLine = ""
+fileName = "eceuropa.sk"
+files = ["train", "valid", "test"]
+
+def listToString(s):
+    str1 = " "
+    return (str1.join(s))
+
+for file in files:
+    with open('raw/'+fileName+'.'+file+'.raw') as read_file:
+        for line in read_file:
+            tokenizedLine = tokenizer.encode(line.rstrip())
+            with open('tokenized/wordpiece-tok_'+fileName+'.'+file+'.en', 'a') as input_file:
+                stringified = listToString(tokenizedLine.tokens)
+                print(stringified)
+                input_file.write(stringified)
+                input_file.write("\n")
\ No newline at end of file
diff --git a/WordPieceTokenizerTrainer.py b/WordPieceTokenizerTrainer.py
new file mode 100644
index 0000000..e46a80c
--- /dev/null
+++ b/WordPieceTokenizerTrainer.py
@@ -0,0 +1,13 @@
+from tokenizers import Tokenizer
+from tokenizers.models import WordPiece
+from tokenizers.trainers import WordPieceTrainer
+from tokenizers.pre_tokenizers import Whitespace
+
+# training the tokenizer
+tokenizer = Tokenizer(WordPiece(unk_token="[UNK]"))
+trainer = WordPieceTrainer(special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
+tokenizer.pre_tokenizer = Whitespace()
+# files = [f"raw/eceuropa.{split}.raw" for split in ["test", "train", "valid"]]
+files = [f"raw/eujournal.sk.raw"]
+tokenizer.train(files, trainer)
+tokenizer.save("wordpiece-tokenizer-eujournal-sk.json")
diff --git a/all-in-one.sh b/all-in-one.sh
new file mode 100644
index 0000000..424c11a
--- /dev/null
+++ b/all-in-one.sh
@@ -0,0 +1,24 @@
+NUMBER=1
+CORPUSNAME=eceuropa
+
+wget https://lindat.mff.cuni.cz/repository/xmlui/bitstream/handle/11858/00-097C-0000-0006-AAE0-A/corpus-en-sk-plaintext.tar.gz
+tar -xvzf corpus-en-sk-plaintext.tar.gz
+gzip -d corpus-en-sk-plaintext/ec-europa/eceuropa.en-sk.en.gz
+gzip -d corpus-en-sk-plaintext/ec-europa/eceuropa.en-sk.sk.gz
+mkdir -p $NUMBER-$CORPUSNAME/corpus/en-sk/
+mv corpus-en-sk-plaintext/ec-europa/eceuropa.en-sk.en $NUMBER-$CORPUSNAME/corpus/en-sk/
+mv corpus-en-sk-plaintext/ec-europa/eceuropa.en-sk.sk $NUMBER-$CORPUSNAME/corpus/en-sk/
+cp $NUMBER-$CORPUSNAME/corpus/en-sk/eceuropa.en-sk.sk $NUMBER-eceuropa/corpus/en-sk/train.sk
+rm -r corpus-en-sk-plaintext
+cd $NUMBER-$CORPUSNAME
+bash ../pre-process.sh $CORPUSNAME
+TEXT=$CORPUSNAME.en-sk
+fairseq-preprocess --source-lang sk --target-lang en --trainpref $TEXT/train --validpref $TEXT/valid --testpref $TEXT/test --destdir data-bin/$CORPUSNAME.en-sk
+mkdir -p checkpoints/fconv
+CUDA_VISIBLE_DEVICES=0 fairseq-train data-bin/$CORPUSNAME.en-sk --optimizer nag --lr 0.25 --clip-norm 0.1 --dropout 0.2 --max-tokens 4000 --arch fconv_iwslt_de_en --max-epoch 1 --save-dir checkpoints/fconv
+fairseq-generate data-bin/$CORPUSNAME.en-sk --path checkpoints/fconv/checkpoint_best.pt --batch-size 128 --beam 5
+cp data-bin/$CORPUSNAME.en-sk/dict.en.txt checkpoints/fconv/
+cp data-bin/$CORPUSNAME.en-sk/dict.sk.txt checkpoints/fconv/
+cp $CORPUSNAME.en-sk/code checkpoints/fconv/bpecodes
+MODEL_DIR=checkpoints/fconv
+fairseq-interactive --path $MODEL_DIR/checkpoint_best.pt $MODEL_DIR --beam 5 --source-lang sk --target-lang en --tokenizer moses --bpe subword_nmt --bpe-codes $MODEL_DIR/bpecodes
\ No newline at end of file
diff --git a/idoc_install.txt b/idoc_install.txt
new file mode 100644
index 0000000..f641d67
--- /dev/null
+++ b/idoc_install.txt
@@ -0,0 +1,13 @@
+conda install python=3.7 numpy=1.19
+conda install cudatoolkit=11.0 nccl -c nvidia
+conda install pytorch -c pytorch
+# https://github.com/pytorch/fairseq
+git clone https://github.com/pytorch/fairseq
+cd fairseq
+pip install .
+cd ..
+git clone https://github.com/NVIDIA/apex
+cd apex
+# https://github.com/NVIDIA/apex/issues/1043
+git reset --hard 3fe10b5597ba14a748ebb271a6ab97c09c5701ac
+1904 pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" --global-option="--deprecated_fused_adam" --global-option="--xentropy" --global-option="--fast_multihead_attn" ./
\ No newline at end of file
diff --git a/pre-process.sh b/pre-process.sh
new file mode 100644
index 0000000..04cb968
--- /dev/null
+++ b/pre-process.sh
@@ -0,0 +1,86 @@
+CORPUSNAME=$1
+
+echo 'Cloning Moses github repository (for tokenization scripts)...'
+git clone https://github.com/moses-smt/mosesdecoder.git
+
+echo 'Cloning Subword NMT repository (for BPE pre-processing)...'
+git clone https://github.com/rsennrich/subword-nmt.git
+
+SCRIPTS=mosesdecoder/scripts
+TOKENIZER=$SCRIPTS/tokenizer/tokenizer.perl
+LC=$SCRIPTS/tokenizer/lowercase.perl
+CLEAN=$SCRIPTS/training/clean-corpus-n.perl
+BPEROOT=subword-nmt/subword_nmt
+BPE_TOKENS=10000
+
+
+if [ ! -d "$SCRIPTS" ]; then
+    echo "Please set SCRIPTS variable correctly to point to Moses scripts."
+    exit
+fi
+
+src=en
+tgt=sk
+lang=en-sk
+prep=$CORPUSNAME.en-sk
+tmp=$prep/tmp
+orig=corpus
+
+mkdir -p $orig $tmp $prep
+
+echo "pre-processing train data..."
+for l in $src $tgt; do
+    f=$CORPUSNAME.$lang.$l
+    tok=$CORPUSNAME.$lang.tok.$l
+    cat $orig/$lang/$f | \
+    perl $TOKENIZER -threads 8 -l $l > $tmp/$tok
+    echo ""
+done
+perl $CLEAN -ratio 1.5 $tmp/$CORPUSNAME.$lang.tok $src $tgt $tmp/$CORPUSNAME.$lang.clean 1 175
+for l in $src $tgt; do
+    perl $LC < $tmp/$CORPUSNAME.$lang.clean.$l > $tmp/$CORPUSNAME.$lang.$l
+done
+
+xd=$CORPUSNAME.$lang.$l
+
+echo "pre-processing valid/test data..."
+for l in $src $tgt; do
+    for o in `ls $orig/$lang/$xd`; do
+    fname=${o##*/}
+    f=$tmp/${fname%.*}
+    echo $o $f | \
+    # grep '<seg id' $o | \
+    # sed -e 's/<seg id="[0-9]*">\s*//g' | \
+    # sed -e 's/\s*<\/seg>\s*//g' | \
+    # sed -e "s/\’/\'/g" | \
+    perl $TOKENIZER -threads 8 -l $l | \
+    perl $LC > $f
+    echo ""
+    done
+done
+
+echo "creating train, valid, test..."
+for l in $src $tgt; do
+    awk '{if (NR%23 == 0)  print $0; }' $tmp/$CORPUSNAME.en-sk.$l > $tmp/valid.$l
+    awk '{if (NR%23 != 0)  print $0; }' $tmp/$CORPUSNAME.en-sk.$l > $tmp/train.$l
+
+    cat $tmp/$CORPUSNAME.en-sk.$l \
+        > $tmp/test.$l
+done
+
+TRAIN=$tmp/train.en-sk
+BPE_CODE=$prep/code
+rm -f $TRAIN
+for l in $src $tgt; do
+    cat $tmp/train.$l >> $TRAIN
+done
+
+echo "learn_bpe.py on ${TRAIN}..."
+python $BPEROOT/learn_bpe.py -s $BPE_TOKENS < $TRAIN > $BPE_CODE
+
+for L in $src $tgt; do
+    for f in train.$L valid.$L test.$L; do
+        echo "apply_bpe.py to ${f}..."
+        python $BPEROOT/apply_bpe.py -c $BPE_CODE < $tmp/$f > $prep/$f
+    done
+done