From 8335c69a4ab1e44ba1ea40fbfe01e116031c8a68 Mon Sep 17 00:00:00 2001 From: jp170na Date: Wed, 4 Nov 2026 03:18:03 +0100 Subject: [PATCH] =?UTF-8?q?Presnej=C5=A1=C3=AD=20v=C3=BDber=20projektov?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 10 +- packages/claude/dist/packages.json | 4 +- packages/claude/dist/zpwiki-graphrag.zip | Bin 7270 -> 7424 bytes packages/claude/dist/zpwiki-rag.zip | Bin 7358 -> 7553 bytes packages/claude/graphrag/SKILL.md | 6 +- packages/claude/mcp_server.py | 9 +- packages/claude/rag/SKILL.md | 8 +- scripts/build_claude_packages.py | 2 +- test/test_work_scope_selection.py | 152 ++++++++++++++++++ tools/answer_evaluation/content_selection.py | 51 ++++-- tools/answer_evaluation/evidence_selection.py | 12 +- tools/answer_evaluation/presentation.py | 9 +- tools/answer_evaluation/question_intent.py | 12 +- tools/answer_evaluation/rag_evidence.py | 26 ++- tools/answer_evaluation/rag_preparation.py | 21 ++- tools/answer_evaluation/verified_rag.py | 23 ++- tools/answers/answer_delivery.py | 8 +- tools/graphrag/evaluation/reproducibility.py | 1 + tools/rag/llm_context.py | 2 +- tools/retrieval/question_constraints.py | 11 +- tools/retrieval/work_selector.py | 79 +++++++++ 21 files changed, 385 insertions(+), 61 deletions(-) create mode 100644 test/test_work_scope_selection.py create mode 100644 tools/retrieval/work_selector.py diff --git a/README.md b/README.md index 35c304a..3c628c3 100644 --- a/README.md +++ b/README.md @@ -1160,7 +1160,7 @@ upozornenie na neúplnosť, nezamieňať autora dokumentu s autorom práce a pri sa nevysvetľujú odhadom; verzia dát neoznačuje verziu programu. Tieto pokyny ovplyvňujú správanie Claude, jeho finálny text však náš backend nemôže programovo vynútiť. Po aktualizácii už nahratých Skills nahraď ich -novými ZIPmi z `packages/claude/dist/` (verzia 1.4.0) a over správanie v novom chate. +novými ZIPmi z `packages/claude/dist/` (verzia 1.5.0) a over správanie v novom chate. ### Nahratie Skills do webového Claude @@ -1513,6 +1513,14 @@ Pri názve a roku práce model dostáva typované fakty a stručné zdroje. RAG nepoužívajú grafové `record_id` ani `work_label`; každé tvrdenie musí mať doloženú wiki URL. Označenie zdroja ako `S1` nenahrádza citáciu. +Obsahové otázky podporujú presný nadpis práce v úvodzovkách, číslo projektu, +nečíslovaný projekt, vylúčenie konkrétneho čísla aj otázku na oba projekty. +Každá práca si zachová vlastné dôkazy a pokrytie požadovaných aspektov. +Odpoveď „obe“ vyžaduje doplnenie pôvodnej otázky z predchádzajúceho spresnenia +v Claude; stateless backend si konverzáciu nedomýšľa. Nadpisy úloh na ďalšie +stretnutie označujú plán. Otázka na skutočne používané metódy vyberá záznamy +o používaní alebo implementácii, nie samotné pokyny na štúdium či návrh. + Pri obsahovom porovnaní viacerých študentov sa pre každého načíta požadovaný typ práce z rovnakého SQLite snapshotu; všetky podklady naďalej podliehajú spoločnému tokenovému rozpočtu. Zadanie a zaznamenaný stav sa kontrolujú a zobrazujú oddelene. diff --git a/packages/claude/dist/packages.json b/packages/claude/dist/packages.json index 3090d27..e5fdc26 100644 --- a/packages/claude/dist/packages.json +++ b/packages/claude/dist/packages.json @@ -3,12 +3,12 @@ { "archive": "zpwiki-rag.zip", "name": "rag", - "sha256": "0c4b1a65d7ec97e89505000c7f60239ae220098ba11dc8a6ae9fca8be396f151" + "sha256": "8e5ab92284ee6f53d10e468cb4edee6cfd354ae869583673cbb6aea8ea9ca437" }, { "archive": "zpwiki-graphrag.zip", "name": "graphrag", - "sha256": "cbfd609ec66c24a03e4c75b33e5a92438a1131c25019df8f2ea522bc917ba5e3" + "sha256": "99a9ea36b439d6b14edc35c2d4d0c1c2cf97139f78e8fd73b8ce2347d82b1092" } ], "schema_version": 1 diff --git a/packages/claude/dist/zpwiki-graphrag.zip b/packages/claude/dist/zpwiki-graphrag.zip index e4b6b6cbef8938a6b7d47ca12eb3fe382043b067..63dd60f35a1063ccc34f9373827884114efdbffc 100644 GIT binary patch delta 2761 zcmV;)3O4oTIDk5TP)h>@6aWAK2mk;8AplUCZY<#l008$A000pH003ulVQ^@2VP`K> zOG!*jE^TCWSkac-wh?{jSK#EO$*Dwd+oTWPy60?M$Hz^xyLIf7HZLMXjwobFfCfNm zm9Oue0YGWj?vu472+Z7>x%a~B^}4cpq?fAqdfI#4R~?0a)|)9BXJ4v+&*V?)*L(Gg zc`#{>@-%3E8}?=jD;4(IQ>UVvOp8DD9QV%S$ISG|a zb6&kD=jOMzGqShZVj=IU;+|AMh-xAqoXs2K)6TW?NH*5h8d+2~HXO-6iB5f*DWvUo z-i@-=U=GB83ElMgG-b3>#>%@nkpDwagSP#wd#bAso^({f&AufLF7h`zC_0#qY)dO; z>3~D=$k-75+{$US`iE22MLgW$?Px+Uw$JcU*Hfsg>IbCIoYVhXiS_RdFeT0%5k-2$ zfxC!ZBzV#t?{xmI;j{Tw`Md&T&F|9@_1Y|pYIW~_3CSUKj`ws1x>oOY2zB`!MHvQn zqkLlMGJD7**JLL{BZX@Pv)fWeUAG zn9D7HIS@R4)Evr6O*%yx@H?0RB_9|t?~#PkGA=-;>kvKnwX~vllO(gC`aJ+ahU&2JKyc_@-HxEZs+Mi(_x2B)n94?^BjS#>ZT%>7fn39bdN9qiE0dJ6KOELmZ z7zzO5iOaK>GIHMZ#_FNk&BJi6GUKWz-dxo>T-pUjZt||j$KXX8ob3ZM zBZlrem7W9F_N2DDePHV7XVYo$lduxzLAf1%9&(Fh9?Be+3NWH!4BCI0Tb1XBnT3#oq%F8k_>0e#hEtwQa8MCQ<4 zo$-?2spd_8WcZ%-W05LRL^(lyJ?cj@&ZGM7>IyuAGPGdE@XgE5=te?i!da;ZW(Eom zbi&(?@r=~tU>TF5tXa%aam3h}9mFiLkG3cvMC>nMSK4Xxjv8H;=#F_l@kJxz(lC%y zq;7+8*>uDL>`$*Sumq*c%!O5d=@X7{w#dTvOe)5p*2GEuXkt}j( z)1ZHRr!b?_4FBVr0agWj6K#FKO* zPdr%g$k~?9IBsT!e89q=%~_wH5OQ8;-6rD+3DYMACCn2a%*x39dg+pmun;@8Ti2+( z5Pz7r=+QW%U1uw6k-`=6=+%R0bzEsIle7>Ms7kp&Pg= zmbOMp*%;f=;|5I;@UD@6Qr&Q$FB5|kXZv8?(XOA=eK0}zEUOhg?lo$Q_NsXU=s7Eb zi-^<&-W3LNG?7yg1z7xw?m?}67PM4v6P6ZR48%yjXGTS|Do4vc?*6iU#I>Rx6kb+q zm0Z(Vwv%lFQuvY5zU0~ zmA<2#huWgc%QRtLw)kex$0Buo3}Ix(6VfD5f=Q`3T*kZ>8G4i+Z+4a3%v ze-y)a2}+^3O`^DeMzI)1%n>C!ojre9hFd{e_{`d4X)xs-4--=?Qbu?th?^A_7=Z+% z!C5%WP`-_5k$Bzds5z+5MmKqh3%>#^kx5suS-ai3$avA`WcS=vgo|XQkQDpGW!P#!*0FMw6|@F{Ztc5Zf6TD(qv) zq}~E*IQD?im8iIk8ok=9oq0t3TtqIDnkW1NPI$&zm+iyrR@pxXw~I%%hCH1?Y)I9t z*Lwo0`zGCg3iz%Y)14|L{)XG_t-~zwg&NC6Tl2_LliMN`Vwi9b6IHBo(|TIKEMtMo zE~Z?DQ`=(a)dzGhuta#bcOT@%B(Pvs*+4H;bzMQIXh>~!vZd(*<`w{O*0^SUZwgm^ z;nx0wJ;;kPO{=8gCQ>EkOL2$fZIVltb8Gc6@xEeza3g~&>YXm#TeRMo1Pb=wh{^^u z4&QC+>P_*n%`^+iQ1@!5hrl$L!w8c4-gKzl^~KWy8t1SZ_`fDRWha6}tLi2>Lv$a! z&ad~Io40RnKWsj}|5u~_c=6!o{@v!sH~(#f1I~|M-`)Q3?rozcXY!5BF{OHQKx0+^ z{Q2&G7U+W`zP`Wv`Q3li)nBi!zDaJi$<}l;`+*fzPi6xH#|Ow|2o-B^GbY5o34AN+ z0_5~oz@l*rcv=`-h7vd=V=E?S_2eqd`XTei>F-@w=voNnbh$^h_-DuT%x0cH%3-e3 zm>0u}rk$|p(+rO*=JJbolVYiUp=~j2`S|lS{^h@E9PB0s$c>DCOopvB0FRY4`L{U1xl&t^0POX z*1J6b3bxZmE0hM*0-hY!Y4Om#Xs^(igGIrh`)y}mr}_eF3f8zY3U*0#zikJ5=60i9B&(oR`ptIB!{9#o}t1)u|h)1ju;Yy+4% zLW;vQKEf$-l44j^I8v>JLQ{KPI+BV=IbRA{Sz)B&=K}G#&^lEL;uMZ;uZf%j4q*Sn zFEgd(iw#mauqHT;r?um54zD4Q1Vc3-Lt1F58#NrXwWcaS0RRB?0RR9N00000000000001_fe;CksT)E9gbI`G8%6@h8k0#H P8@6aWAK2mk;8AprB*yB0_Y003 zOG!*jE^TCOSY4CbHWGa2uR!JHk~&G=yG!oDdrsBHaa^0+?QR^q^6?@<O^ip+Rk2|lss-@6>cr!-h>`V3YME<1ycu>EZ zy-917r(W~hurp&=sIb$XS{2=B8vN8#+&Pawj?V8@JdK3EPo#ZN+P123lZ|1eJ_J%* z=hcgHZhmW9BYUe27V@qt?nniMs7CU^*}O46ZCx`BWaFw@B8%$Ah9mhW(W#FUg|yw) zyFs?<&4D<7p_>k$ri>QKSa~-E@_!4e*S4E7%rpE`R8DC z-!Bl-cuGm?h*;7tb68lo0k1;_!_(jnjD)i|z+m$XP1U$zfRttmJYgeLKILQb_uM@a?Hr2+Q>NYGW?qQxTw;h;3Wz& z^+zt^S`9klO-L^a3M+NP`jmGH{4!)3*y~My&!}-pttv>Q=@Ho0G!m^WBxT7YZtCd7(y}GlhYewMeW1 zCbT7gUG0g$O5Zxf)m1KUeCWw?7&dmaGf}LP^wRw*ZFibEBI*7rky8WX#!U^>BifvO=+5 zLM3}J-dQ%iMWvMTlU4PYRwdI8FNEwBO6iNw2J}Up9THX`<_aoX<0Ze-iZ}g<;d|y1 z#URCp5nEb1?Q zo?8eQM~to6!a|eywFQB&&VQk}X{(_fHMlla8@o%AYb};mGmvAXb`3qQTVmVyr&q9O zg1_d%!K(BLM>ttzVLR5eu}wdT(L$2X>XXs5Oc5%@fGZRbJOK`B>8Q2mxVGHnufe?dPM9{(Xq+I|$;w(} zFp^_~!2VtDHo!Ky9Kii%rQQ)+4|puh;N%({TNYM(@dJmh(aYIcYoW3>wx!24L=o_= z7OHMo=X1s2#L4chJKE(F-g^_oakE*{<4%KH$ZN$Lpq(I|=prIDfp^7!*fN;NsYtJy zyQJJDQoh+_a zJVr*;5v3t6Pnu;JX3@0p*+7Y9)%<0GIwOHKpxoUW|Zd?>tLOrl&&Ol8H-JxsVZJ z;)=&9jHvF>O)US;?Zgb?V6Nw_Gbxg004ovO+gUE0RX*|HdfYdN@~Ah=ZRLOjll9>PbfFiBfL4aXi( zx)K%BsG-$PZOtR%XA+qvHI4WOp76}IF6G1P7I}ONZX1udl_{>yAY({XFPA$4s=GQ3 zrF_GRX_61gzgwd_>rhL4jl**3t+`~W&aw!_7)DJ0(iN+JELu-9n58Um#X^-!aRUB4 zO{(`$FR(;-clYn*#VD|#R*ltXtGcexsHjh}I@u!nfVu?$oHeHW?@aNk&)gMX@C@>z zRMR5$a1*H#`cm8>%{DsD=i+jjd|xrRk-?SjovwNtNN-3D3Ku>jT{b{CJX)`+H)W2k z$4N|v+E-hD-3PY897d4Tccuk**B4I~g=L@l?Jc%Gt+;7ul2=!ss`Qu@I^Y+c% z`}K!+|E<;EFCN@H+^&Cm^Ls5GaDM#y{_cm{x3wCb$rlF4mg>y`!m57!<^B%n!y~?Y zxc}w$H+A*ztE;clv|8uUbUpc=JE|Vd8Uq|3Apc2!0LE%u%m}fsG1+2d3}~l!0u~J$ zz|&yBWhj9|8f<0ASvlht_+uaslru}4c@6aWAK2mk;8ApjvWpV2>)BMBpaK~BRk5JmSrMU*uPcI?5+*&kl>Lj43>fLct5C|c!=Kq;DqwnjkC`xDHjUsl5E^|nP*YJ8-~(Z2e>lYlhB~NLU7DLOR5%vt5zyUtwl~=7^^@!#@7L^^tK=~ zM@V6q#z#1Xd`UGdD;%j&OrmbA;)aAIlE!i->XvCC^D~|-3h#{MjI7os+g1@72?*H! zg&>Q;DUo?)5Mw=JDM z#L&-uH!Rcf99OoeIJ+koZU0Z!BOIx>UH+djY!iHwf9jG~vSQ`4fC^|43iH~#7DxyH z0BsSI@*6cAAv2%RKLG#$^#K3?761SM0000000000q=9z`lPn1slUp1r0_q8qpb8w5 TqZ}#$EtAe1ECxOs00000*qrCB diff --git a/packages/claude/dist/zpwiki-rag.zip b/packages/claude/dist/zpwiki-rag.zip index c75e00cb39f30060a644eeeaf9adacf70dd21d19..7f1361d64101df7eefd2a2646eb1a9c2706510fb 100644 GIT binary patch delta 2890 zcmV-Q3$^sVIe|NWP)h>@6aWAK2mk;8Apndui0Wer004^=000aC0044fXD?GrNlZ*G zZDeg&U6b3m5q#&bK>6j8ilp6qTpqk}s+5iG+I;M)Y+qgC7ZD;M5w{?~13+n&U!R@< zKyrQeaOD$805d&3J&j$j*OisMdMms-O7zgH}vpQ`DsGw!5S;I9YoPF{tu5kCYaj?Rl4945T_c~l{$ zZ-N_rqt*x6JgZh{+v>A!M`^@SHZk0Z5i3t#4jrweExa<4;tZW0(iV*yZ48@gjg+db z4HxApP+IDLzYBs7!vb;iuICp7e7Vd>dm80QZK^L`$%u2K*XoQQ zFN6v~AY>2<;VRpxm9Ti}>5;Y}`my04H&yikLD!u4UrH!{tydzbglFFA7<1G{$yVD_ zT2j+~^mIxhx~l%)Kq2hu{YLD7u<}TVj=us7KuBMIZp6b;Shq;u8RIUflMK&{q_aql zDigb0({)ItkIotJ0tZt}(u$8%flu3y;?k)ZC81shqZ_c`igb)Z!E)F+ZHaLD!KgSV zu)?F^nlhCU&(cm)lt|)$`t}(SuX=YTIGn{H8V6*2D(u{&`Wx|ghcu}^L>0t}6pUVO zNrB*h@r$HTR$`EeP67WPv;hSS!R9^U7m}L-7_}{;Ct}m`*1Lg|nXvtv0)k33(ijAT zq&n3+3t9_+FU~)Ucpc{6sH{3A)@H``h(ZVceUd~NMx+@L1t^JK0#UCVGf<6fBpm27 z{2EPMRq0R^C^9lNGWOS6^fKa2NG}Qs8?hCCy$<|NpxgwR1orY^2sLV&O$Ej@27w)Q z!_o4GO0}v@tOy9TZhVD1n4G`cY?pvpuDo%2+rVd6^3M)$pHHMoE` zNVHK*f5&130P)1-hDsSZuTR=aQyoWR<|-4ea-i**DztMxhmxPVgq{z|2Vx@CW>GDF z+M>$hgf^FyEp$bENWH=zu2_%=ZSbifcpZJ)B~sooC~pofc5hZi=LG$%Xfa6x@5F`T zh@X`jC?@SEB|+Fk(WoYBm#*wMhBo9Agf!g3s2XQAVOKD@$sh^|4vkixr8ZRWSpu-G zf7R$GETw90s(a?B%7lxfx5%?lY$h##7y@XWAz^%75SCNat3jtF7W81PMh{N9A#xK> zz+~dD1Y&9FLy5en9Sbm|1k~28@^=*@2tv-33*w-F%!vu+zS$$XlcIE?P`_u1l+&R5 z_x+AGv8XzNMPduUi1K17yqE$pmE;!^x2gWH5l?i;MU%)NRw;JZ*f~-m)r=f}h_Ywx zo$aaLd9Q#sUa=P0BihIV(L9@kY#JZ~N{zM>LexXrEe-8(!&TWpR^R+;KyT_iHHrT& z6)ik#V=(4g?K$7;hBp4P95qJwC=tH>{`M9E2k09}a#%Few|bXaDz##&2pShmq3Mj> z3!PE?m}bL46RT*{)5xDHhL@`b(Ysd7exklfLWZgM zBh9IrU>ne%dvJZ_< zQujfyKN*Z{l16~yGTQ(K7g!Rl%F$V4XS=Y)?j?$3We7_NYpA^fts7p6s=NdR$Og7anegO783?9wmnweV@>t6QAJm6hj z`U0|_T=GE`FDOka7)()C&@yu-vQSWA94I3NuabGTZydIV&@tf_vQ|z=#LHU*I41b? zdfEpgUg6z@Np{c0f~uEJj)A(JW8Y<-(~enB(D!AOQvw>|GShy4=JznL_6XQyaHT=D z3z!yaM0;PPw^?T8oM$Qr>Y9fB;&LH=8fhS^M$#CaV4P8lFj~ppq@H=?j3(fvT?9cV zjP}{Oi(M}R%NtPpvhOQsK+r;53Q-myXyEav*m!a2MJ|0{Gta~I%1yvHQo~?PuyR0p z*st0xyZM02NDFFztMIm3i{!3nW0#CIsEOUxnN2LCiA&k%o zsvh5M5aC&~`=b6%?4f&+*uH{|V4$KkKnIc=NfBPxZmIFoUiqydsc7Vx?n0#pHFms` z85gf@0$&Vocou;F7mPW!huNvpGy0r;8XHoIhJ2m}`QB%L`<}zc)`*k4OzQJ%Gz*a5 zk>5-Wm4+54UAL|oNs#t(M7R(|>Xt~Fm(EXDPf*utbJ>DTu<7U?NJ)Ai@Fd< z)2LaFr#zN_)kTfHkp4ot78iCvwpe}SET z(vbWY5PXs?O8VnLo$DQ{S^Iu5*(BQGvrMWiM*3LDl2LPc%r`mTsYwM|fD7;R$ zNac=yGRT)g#62o{1^qE_E-etu=JUz{g2O2j{phn_?z9+=ZF8$)sI6Xb-(@d z?d@-q_pb9`+>hRnH;{wgKdWn9Q4!NE3OLt)QTGZJ+6TImwgEmf&8LB@h6U-;Q|$GM zBSF_ik(GYFB9tbDGa^?MDDFX`_vjt~au8x3jT_ryFLPgczB%)${D_HvaZ(ix|dg*GCI%{D)*fqw~b_d)HA=Uo}P)h>@6aWAK2mk;8AppUY-bXu= z+6W$hOK!t33`F-iMc{Seim|@ELDB4b0D}^#L}=_dvfUyGa`!4ui+-RLX9iEi@UmL3 zE05ugdulVE=9HPLr}dO)ez3)zho5H7=Px!pB{s@Rl0cxPv=V*ruGtu^9T*=`8;MZ}K(-D$ zt>&`7ZZukm%7Ls3`nt5!r9O3Y*AC}tf9!AUrUr3uuE*|g&12Yehi$pan3gas^3PmB zt!h!NuB%V8Gzx|h3XC;~>SGB20E-lp#2YXi!Ia)dI{^Ry>j3}&5dZ)H0000000000 oq=AhIlMoy|0{04&Pzo25W*jO4H5-$T8WxkN91sR=8~^|S0IWV!00000 delta 2696 zcmV;33U~E^JH9!8P)h>@6aWAK2mk;8App!308O0;004Xv000aC0044fXD?GrNlZ*G zZDeg&+mai%5q+PpK>5Wl^Z9B`(|fe8W=07EYQ^*-Ie zT%`DEmjux0K7IOhgX{IWa{5fKRnW((rPu`1C*$2~^^tUcXgc+It9~>mqhcCba!F~| zs?qwHWA2b_A_>-&n6l_xD+?l8X;8=E?z0H zqkJ@c8<5SU2+Yrl*3}+uZ)W+42?LssjBbcbS;7ZWpw4Q^-4v`cos{CRJ^vNTjP%TNX_>6 z+bM|TtNQ1jUD)yZMs0yG1q8&`-v9<6xNFT(fM`YZCJ<4KV84#9C|Qde@i zjyd(odkbFRV)jWp^`0~E;o-fyb~K};=%u%&0Sj&j#}X1&+%91km$bDjIM&~CfI7jW zY+!N@8}~@-xd0L9>@SRhcd%_>x4=?0{(MH#H|p;m(Q#UsA#f3%TeCEY`}yk^eeNyN z)9cKCqKN;avB9ZVa3m4I83ff)b5eX3xL$oY zsdVe-lBgUVGg%8!&+x)T`FzyO6G5aI;02fnE0GyDUej}GU1A(~8$Jy-t*U$}@{$CZ z8WD$EtOe5aPMxvJ5OqSSF zBa0T0F>}}(SdEe)z=(p;3A>JJd-fJkx9q`0FnBRCu1#i ztu;a7y=~a=r=_-7b4H4=_q)40P#e&HHz>xq(3r28o{28xE*XLJf!Hj^uwH63BKb)Q z2}E&-lC_;Re##)?4L&j}TIvx$W0bXsVCj#jb~0XVqcFcY#`zO@?d7iW+O+zrM4(?&^4 zi7+?BHxz;ob#~+@WEMxqz*katTKXC^VG7F+sUIZKgPSOS&mQLV4#9s{CPUtWqoRI3qr4 zA*X=@IA|Mz$#)y|fmo)Js>f=8f3|+AFMkACSEar5z#9#nAMXV zh&y3z!laB#JZFnVPf+)xh;sl2;xf~I=6BzV_9)Cn;7WsRk6=2e z5!=4%;EK%ZInJC89l6qBAA7y$wFmN|10>P0B24|wvJ;H;% zYP+oFJ*LkV)Ta2lTB|%x%`H7oQlKWR)6_f1)wxT5nSoJc(6*5WZ4rXhw~lduPH^^Q zn1zQIBYl_aJF|xeVfF9@YzzY@tpPfa+{B9TW$u<5uiZ^nElWjzqc1%Cjx}gt@y27A zDicW>0t>pl1YSum0ROLOb21o|8HK;&?J~^CD8k;L4WXlN)Vngc758BpeoB>T>0b-H zMj*fCNh23jZdwv_-TG!=LE7m7?m`r~S`uqszCXV;)VK{+^2P~%;|{s1`H_>?_?fYS zPe26dm?)ynGYY{yXe^^i2SH`2< zU>A8|kfv6%Y)_>v>AfiCqDG*}7G!c@RPN#&gaff(Kn}i(oh^ng-VWT1$^Z?ace*?w z!c;pG3CF=Zca@lDtSItyy7|pSLeYW zb0-|l8d9FT*iuH6{93F_zGF3O-~V+sEn|(9nUfpYxDh0Ldj7;ofGz~jZ_k5qSjc3$ zuh6*3W@6lbT%oOCVMGiEH@5tZ`Y7Cw7|Ej$W64*%k(#-a80Q??^m!LlulS>OkA7q#vJEM?2bwK#osrvJF_wfGR)2H1JAO2OV z-z~m)*go$5`R>29l-vCE+vlh6AK%wi?@f6cc^R;O%!R4_kB`q!014&zo9*+*$N#9F zSrW6l-~IXS?zegNu1jOw4Z-qkpnJ1B(XFmHiC$V4aIWzpC1hxaQ5~ad#GIM?Q!iP= zf&8PvI9{*B6I5MNMd{}!!dRr3jCd-F?02tGd;AUnIT$e>laP?{>7E}53ai3iZV1FZ z8}B)5sa{Gvh*v-|@e>P$j53ehQ8|iv_Twwk(97J6=|9hGZ>=BNal9;b73AqO{|Hi9 z4^DO-T@>%g&yQP~UBS{Xf8@wN@6aWAK2mk;8 zApmv8toA;Wpa~v-O>W#U42Ab|ih92k1I1^VxG!>Da}jRtk4stQZu3|Yt*50Q&n#7MUZmudg?pOjKT?`K zB-3M_cDvByhqRR2<(#KwlYjsE`S8$={O+ZElDW#SbvXWiXV##zYz(p?6oZiw6FPK0 zQY2QU^RA1|2zmen(4Ezg0uFLIQFclTjNMlWQFi2C5qX0001N C4Gj_i diff --git a/packages/claude/graphrag/SKILL.md b/packages/claude/graphrag/SKILL.md index 40f4b0d..c0d9457 100644 --- a/packages/claude/graphrag/SKILL.md +++ b/packages/claude/graphrag/SKILL.md @@ -10,12 +10,16 @@ For `/graphrag `, send the text after the leading `/graphrag` as the q to `zpwiki_graphrag`. Preserve every condition. If no question follows, ask for one. Use this graph tool rather than RAG when the user explicitly selects `/graphrag`. -For every call, copy the latest user question verbatim after removing only its leading +For a complete question, copy the latest user question verbatim after removing only its leading command. Do not reuse an earlier question or an example, paraphrase, silently correct a name, add a year or drop a condition. Before calling, check names, work types, years, negations and comparisons against the current message. Before presenting the result, check that its `query` matches that question. A mismatch is not a usable answer. If clarification is needed, ask specifically for the missing or ambiguous condition. +For a short reply selecting the immediately preceding options, reconstruct that original +full question and apply only the selection. Preserve every condition; if context is missing, +ask for it. Compare the returned `query` against that resolved full question. A text question +about methods or progress belongs to RAG; preserve any backend guidance instead of inventing graph evidence. For multiple explicit commands in one message, handle each question separately in order; keep all conditions belonging to each question together. diff --git a/packages/claude/mcp_server.py b/packages/claude/mcp_server.py index 914c3b7..33df4fd 100644 --- a/packages/claude/mcp_server.py +++ b/packages/claude/mcp_server.py @@ -20,11 +20,16 @@ from starlette.responses import JSONResponse # Host-model rules belong to MCP initialization and Skills, not tool-result data. QUESTION_POLICY = ( - 'For each tool call, copy the latest user question verbatim after removing only its leading ' + 'For a complete question, copy the latest user question verbatim after removing only its leading ' '/rag or /graphrag command. Keep names, work types, years, negations, comparisons and every condition. ' 'Never reuse a previous question or an example, paraphrase it or split its conditions across searches. ' 'If the user supplies multiple explicit commands, handle each exact question separately in order. ' - 'Before calling, compare the query with the current user message. Respect the explicitly selected engine. ' + 'For a short reply selecting options from the immediately preceding clarification, reconstruct the original ' + 'full question and apply only that selection. Preserve its person, year, work type and requested aspects. ' + 'Use exact quoted work headings for a selected work; if the user chooses both, ask about both works in one full query. ' + 'If that context is missing, ask for it instead of guessing. ' + 'Before calling, compare the query with the current complete question or its explicitly resolved clarification. ' + 'Respect the explicitly selected engine. ' 'The returned query is audit data: compare it with the current question before presenting its answer. ' 'If they differ, do not present the mismatched answer. Ask for a concrete clarification when needed. ' ) diff --git a/packages/claude/rag/SKILL.md b/packages/claude/rag/SKILL.md index aef6d4e..650c9bf 100644 --- a/packages/claude/rag/SKILL.md +++ b/packages/claude/rag/SKILL.md @@ -10,12 +10,18 @@ For `/rag `, send the text after the leading `/rag` as the question to `zpwiki_rag`. Preserve every condition. If no question follows, ask for one. Use this text-search tool rather than GraphRAG when the user explicitly selects `/rag`. -For every call, copy the latest user question verbatim after removing only its leading +For a complete question, copy the latest user question verbatim after removing only its leading command. Do not reuse an earlier question or an example, paraphrase, silently correct a name, add a year or drop a condition. Before calling, check names, work types, years, negations and comparisons against the current message. Before presenting the result, check that its `query` matches that question. A mismatch is not a usable answer. If clarification is needed, ask specifically for the missing or ambiguous condition. +For a short reply selecting the immediately preceding options, reconstruct that original +full question and apply only the selection. Preserve the person, year, work type and +requested aspects. For one work, use its exact quoted heading; for "both", request both +works in one query and keep their evidence separate. Do not search the words "both" alone. +If the preceding question/options are unavailable, ask for them rather than guessing. +Compare the returned `query` against this resolved full question in that case. For multiple explicit commands in one message, handle each question separately in order; keep all conditions belonging to each question together. diff --git a/scripts/build_claude_packages.py b/scripts/build_claude_packages.py index c24a8b4..9a0042a 100644 --- a/scripts/build_claude_packages.py +++ b/scripts/build_claude_packages.py @@ -11,7 +11,7 @@ import zipfile ROOT = Path(__file__).resolve().parents[1] BASE = ROOT / 'packages/claude' ENGINES = ('rag', 'graphrag') -PACKAGE_VERSION = '1.4.0' +PACKAGE_VERSION = '1.5.0' def json_bytes(value): diff --git a/test/test_work_scope_selection.py b/test/test_work_scope_selection.py new file mode 100644 index 0000000..f3a7ae3 --- /dev/null +++ b/test/test_work_scope_selection.py @@ -0,0 +1,152 @@ +"""Exact/both/negated work selection and proof of actual use, not future tasks.""" +import json +import sqlite3 + +import pytest + +from tools.answer_evaluation.rag_preparation import prepare_named_content +from tools.answer_evaluation.rag_evidence import rag_evidence_context,project_number,content_sections +from tools.answer_evaluation.verified_rag import generate_verified_rag +from tools.answer_evaluation.content_selection import facets_for +from tools.retrieval.work_selector import parse_work_selector +from tools.retrieval.question_constraints import requested_years,unsupported_constraint +from tools.answers import answer_delivery +from tools.rag.llm_context import compact_rag_payload + + +@pytest.fixture +def projects(tmp_path): + db=tmp_path/'index.sqlite';path='pages/students/2016/eva_nova/README.md' + with sqlite3.connect(db) as c: + c.execute('CREATE TABLE documents(path,title,author,published)') + c.execute('CREATE TABLE chunks(document_path,text,heading_paths_json,published,chunk_index)') + c.execute('INSERT INTO documents VALUES(?,?,?,?)',(path,'Eva Nová','Autor dokumentu',1)) + for i,(label,text) in enumerate([ + ('Diplomový projekt 2020','Hotové:\n- Funguje databáza SQLite.\nÚlohy na ďalšie stretnutie:\n- Naštudujte si ako sa používa Redis.'), + ('Diplomový projekt 2 2020','Stav:\n- Implementovaný parser pomocou Python.\nÚlohy na ďalšie stretnutie:\n- urobte návrh metódy PageRank.\nDo ďalšieho stretnutia:\n- Naštudujte si ako sa používa Promise.')]): + c.execute('INSERT INTO chunks VALUES(?,?,?,?,?)',(path,text,json.dumps([['Eva Nová',label]]),1,i)) + return db + + +def prepare(db,q): + raw={'status':'ok','reason':None,'sources':[],'context':'','complete':False,'truncated':True,'truncation_reasons':['result_limit']} + return compact_rag_payload(prepare_named_content(db,q,raw)) + + +def select(messages): + context=json.loads(messages[1]['content'])['context'] + a={'status':'ok','claims':[],'source_urls':[], + 'evidence_ids':[u['evidence_id'] for u in context.get('content_candidates',[])]} + return {'choices':[{'message':{'content':json.dumps(a)}}]} + + +@pytest.mark.parametrize('phrase',[ + 'pri práci "Diplomový projekt 2020" (nie Diplomový projekt 2)', + 'pri nečíslovanom diplomovom projekte v roku 2020', + 'pri diplomovom projekte bez čísla v roku 2020', + 'pri diplomovom projekte v roku 2020 okrem projektu 2', +]) +def test_exact_unnumbered_and_excluded_number_select_the_actual_work(projects,phrase): + q='Aké metódy používala Eva Nová '+phrase+'?' + assert not unsupported_constraint(q) and requested_years(q)==[2020] + p=prepare(projects,q);assert p['status']=='ok' + assert [s['work_label'] for s in p['work_scopes']]==['Diplomový projekt 2020'] + r=generate_verified_rag(q,p,select) + assert r['accepted_for_delivery'] and 'SQLite' in r['answer'] + assert all(x not in r['answer'] for x in ['Python','PageRank','Promise','Redis']) + + +def test_numbered_work_keeps_actual_methods_and_rejects_future_task_evidence(projects): + q='Aké metódy používala Eva Nová pri diplomovom projekte 2 v roku 2020?' + p=prepare(projects,q);c=rag_evidence_context(q,p) + assert all(u['phase'] not in {'planned','assignment'} for u in c['content_candidates']) + r=generate_verified_rag(q,p,select) + assert r['accepted_for_delivery'] and 'Python' in r['answer'] + assert 'PageRank' not in r['answer'] and 'Promise' not in r['answer'] and 'SQLite' not in r['answer'] + bad={'status':'ok','claims':[],'source_urls':[s['source_url'] for s in c['sources']], + 'evidence_quotes':[{'source_id':c['sources'][0]['source_id'],'quote':'- urobte návrh metódy PageRank.'}]} + r=generate_verified_rag(q,p,lambda _:{'choices':[{'message':{'content':json.dumps(bad)}}]}) + assert not r['accepted_for_delivery'] and 'quote_outside_delivered_candidates' in r['attempts'][-1]['issues'] + + +@pytest.mark.parametrize('word',['oboch','obidvoch','všetkých']) +def test_each_requested_work_has_its_own_methods_and_heading(projects,word): + q=f'Aké metódy používala Eva Nová pri {word} diplomových projektoch v roku 2020?' + p=prepare(projects,q);c=rag_evidence_context(q,p) + assert c['requirements']['separate_works'] and len(p['work_scopes'])==2 + r=generate_verified_rag(q,p,select) + assert r['accepted_for_delivery'] and r['answer'].count('**Eva Nová — Diplomový projekt')==2 + first,second=r['answer'].split('**Eva Nová — Diplomový projekt 2 2020') + assert 'SQLite' in first and 'Python' not in first and 'Python' in second + assert 'PageRank' not in r['answer'] and 'Promise' not in r['answer'] + + +def test_omitting_a_second_work_with_evidence_is_rejected_even_for_the_same_person_and_year(projects): + q='Aké metódy používala Eva Nová pri oboch diplomových projektoch v roku 2020?' + p=prepare(projects,q) + def incomplete(messages): + c=json.loads(messages[1]['content'])['context'] + a={'status':'ok','claims':[],'source_urls':[],'evidence_ids':[u['evidence_id'] for u in c['content_candidates'] if u['work_label']=='Diplomový projekt 2020']} + return {'choices':[{'message':{'content':json.dumps(a)}}]} + r=generate_verified_rag(q,p,incomplete) + assert not r['accepted_for_delivery'] and 'missing_methods_evidence' in r['attempts'][-1]['issues'] + + +def test_missing_methods_are_reported_for_the_right_work_without_stealing_another_works_evidence(projects): + with sqlite3.connect(projects) as c: + c.execute("UPDATE chunks SET text='Úlohy na ďalšie stretnutie:\n- Použite metódu BERT.' WHERE chunk_index=1") + q='Aké metódy používala Eva Nová pri oboch diplomových projektoch v roku 2020?' + r=generate_verified_rag(q,prepare(projects,q),select) + assert r['accepted_for_delivery'] and r['answer_coverage']['status']=='partial' + assert 'SQLite' in r['answer'] and 'BERT' not in r['answer'] + assert 'Diplomový projekt 2 2020' in r['answer'] and 'skutočne použitých metódach' in r['answer'] + + +def test_plans_remain_available_as_plans_with_natural_verbs(projects): + q='Aké plánované úlohy mala Eva Nová pri oboch diplomových projektoch v roku 2020?' + r=generate_verified_rag(q,prepare(projects,q),select) + assert r['accepted_for_delivery'] + assert 'Urobiť návrh metódy PageRank.' in r['answer'] and 'Naštudovať si ako sa používa Promise.' in r['answer'] + assert 'urobte' not in r['answer'] and 'Naštudujte' not in r['answer'] + + +def test_unknown_requested_work_is_not_silently_dropped(projects): + q='Porovnaj metódy Evy Novej pri "Diplomový projekt 2020" a "Diplomový projekt 3 2020".' + p=prepare(projects,q) + assert p['status']=='ambiguous' and p['reason']=='unresolved_work_selection' + + +def test_short_followup_needs_context_instead_of_global_retrieval(monkeypatch): + def forbidden(*a,**kw):pytest.fail('A bare follow-up must not search or call an LLM') + monkeypatch.setattr(answer_delivery,'build_rag_context',forbidden) + r=answer_delivery.deliver_answer('daj obe',engine='rag',complete=forbidden) + assert r['status']=='ambiguous' and r['reason']=='missing_followup_context' + + +def test_negative_selector_is_not_a_positive_number_and_literal_title_years_are_not_filters(): + q='Aké metódy používal Ján Holp pri "Diplomový projekt 2020" (nie Diplomový projekt 2)?' + selector=parse_work_selector(q) + assert project_number(q) is None and selector.excluded_numbers==(2,) and requested_years(q)==[2020] + assert selector.matches('Diplomový projekt 2020') and not selector.matches('Diplomový projekt 2 2020') + assert requested_years('Kto má názov "Projekt 2 o svete 2099"?')==[] + assert not parse_work_selector('Kto má názov "Projekt 2 o svete 2099"?').specific + assert unsupported_constraint('Aké metódy používal Ján Holp, nie v roku 2020?')=='exclusion' + + +def test_unimplemented_method_is_not_proof_of_actual_use(): + assert 'methods' not in facets_for('Metóda BERT ešte nie je implementovaná.','progress', + 'Aké metódy skutočne použila Eva Nová?') + + +def test_scalar_question_also_preserves_exact_work_and_exclusion(projects): + q='Aký názov má "Diplomový projekt 2020" Evy Novej (nie Diplomový projekt 2)?' + p=prepare(projects,q) + assert len(p['work_scopes'])==1 and p['work_scopes'][0]['work_label']=='Diplomový projekt 2020' + assert 'Python' not in p['context'] and 'Promise' not in p['context'] + + +@pytest.mark.parametrize('heading',['Úlohy na ďalšie stretnutie:','Do ďalšieho stretnutia:','Do ďalšiehio stretnutia:']) +def test_source_task_headings_cannot_inherit_a_completed_state(heading): + sections=content_sections('Stav:\n- Implementovaný parser.\n'+heading+'\n- urobte návrh metódy PageRank.') + assert [s['kind'] for s in sections]==['progress','task'] + assert sections[-1]['phase']=='planned' diff --git a/tools/answer_evaluation/content_selection.py b/tools/answer_evaluation/content_selection.py index ee636cb..99baf95 100644 --- a/tools/answer_evaluation/content_selection.py +++ b/tools/answer_evaluation/content_selection.py @@ -3,7 +3,7 @@ from __future__ import annotations import re from .context_policy import normalized -from .question_intent import FACET_LABELS, planned_experiments, requested_facets +from .question_intent import FACET_LABELS, planned_experiments, requested_facets, actual_methods_requested COMPLETED = (r'\b(?:(?:implementovan|natrenovan|vytvoren|extrahovan|vykonan|vyhodnoten|nameran|' r'vyriesen|odstranen|otestovan|porovnan|dosiahnut|dokoncen)(?:y|a|e|u|ou|ych|ym|ymi|om)|' @@ -38,7 +38,7 @@ def split_units(text: str) -> list[str]: def effective_role(text: str, role: str) -> str: - n = normalized(text) + n = normalized(re.sub(r'^\s*(?:\d+[.)]|[-+*])\s+','',text)) if role in {'other','progress'}: from .presentation import GOAL_VERBS first=n.split()[0] if n else '' @@ -71,10 +71,11 @@ def facets_for(text: str, role: str, question: str, phase: str | None = None) -> if (phase or evidence_phase(text,role))=='result':facets.append('results') if re.search(r'\b(?:problem\w*|tazkost\w*|prekazk\w*|chyb\w*|zlyha\w*|nepodar\w*|nevyries\w*)\b', n) and role not in {'assignment','task'}: facets.append('problems') - if re.search(r'\b(?:metod\w*|technolog\w*|nastroj\w*|pouzi\w*|vyuziv\w*|pomocou|implementovan\w*|natrenovan\w* model\w*)\b', n): - actual_use=(re.search(r'\b(?:skutocne|pouziva\w*|pouzil\w*|vyuzival\w*|implementoval\w*)\b',normalized(question)) or - re.search(r'\b(?:pouzit\w*|aplikovan\w*|vyuzit\w*)\s+(?:metod\w*|technolog\w*|nastroj\w*)\b',normalized(question))) - if role not in {'assignment','task'} or not actual_use: + if re.search(r'\b(?:metod\w*|technolog\w*|nastroj\w*|pouzi\w*|vyuziv\w*|pomocou|implementovan\w*|natrenovan\w* model\w*|indexov\w*|databaz\w*|webov\w* rozhrani\w*)\b', n): + actual_use=actual_methods_requested(question) + actual_statement=not re.search(NOT_COMPLETED,n) and (re.search(COMPLETED,n) or + re.search(r'\b(?:pouziva\w*|pouzil\w*|vyuziva\w*|vyuzil\w*|vieme indexov\w*)\b',n)) + if not actual_use or (role not in {'assignment','task'} and actual_statement): facets.append('methods') if re.search(r'\b(?:experiment\w*|meran\w*|test\w*|otest\w*|vyhodnot\w*|benchmark\w*|f1|presnost\w*)\b', n): wants_done=re.search(r'\b(?:vykonal\w*|vykonan\w*|otestoval\w*|uskutocnil\w*|uskutocnen\w*|realizovan\w*|vyhodnotil\w*)\b',normalized(question)) @@ -159,13 +160,29 @@ def evidence_units(context: dict, quotes: list[dict] | None = None) -> list[dict for other in units)] +def scope_groups(context: dict) -> list[tuple]: + req=context.get('requirements',{});groups=[] + for person in req.get('people',[]): + labels=list(dict.fromkeys(s.get('work_label') for s in context.get('work_scopes',[]) if s.get('person')==person)) + groups.extend((person,label) for label in labels) if req.get('separate_works') and labels else groups.append((person,None)) + return groups + + +def matches_scope(row: dict, person: str, label: str | None) -> bool: + return row.get('person')==person and (label is None or normalized(row.get('work_label'))==normalized(label)) + + +def scope_context(context: dict, person: str, label: str | None) -> dict: + return {**context,'content_passages':[p for p in context.get('content_passages',[]) if matches_scope(p,person,label)]} + + def candidates(context: dict) -> list[dict]: from .evidence_selection import identify facets = context['requirements'].get('facets', []) selected = []; seen = set(); units = evidence_units(context) - for person in context['requirements']['people']: + for person,label in scope_groups(context): for facet in facets: - relevant = [u for u in units if u['person']==person and facet in u['facets']] + relevant = [u for u in units if matches_scope(u,person,label) and facet in u['facets']] if facet=='progress' and 'results' in facets and any(u['phase']=='progress' for u in relevant): relevant=[u for u in relevant if u['phase']=='progress'] if facet!='assignment':relevant.sort(key=lambda u: -unit_score(u['text'],u['kind'],facet)) @@ -186,10 +203,14 @@ def candidates(context: dict) -> list[dict]: def missing_facets(answer: dict, context: dict) -> list[tuple[str,str]]: + return list(dict.fromkeys((person,facet) for person,_label,facet in missing_work_facets(answer,context))) + + +def missing_work_facets(answer: dict, context: dict) -> list[tuple]: found = evidence_units(context, answer.get('evidence_quotes', [])) - return [(person,facet) for person in context['requirements']['people'] + return [(person,label,facet) for person,label in scope_groups(context) for facet in context['requirements'].get('facets', []) - if not any(row['person']==person and facet in row['facets'] for row in found)] + if not any(matches_scope(row,person,label) and facet in row['facets'] for row in found)] def fit_candidates(sent: dict, context: dict, limit: int, encoding: str) -> None: @@ -200,8 +221,7 @@ def fit_candidates(sent: dict, context: dict, limit: int, encoding: str) -> None if counter.encoding is None: raise RuntimeError('Missing context tokenizer') original=list(context.get('content_candidates',[])) - groups=[[row for row in original if row.get('person')==person] - for person in context['requirements']['people']] + groups=[[row for row in original if matches_scope(row,person,label)] for person,label in scope_groups(context)] ordered=[group[i] for i in range(max(map(len,groups),default=0)) for group in groups if i list[str]: facets=req.get('facets',[]);groups=[];labels={f:FACET_LABELS[f] for f in facets} if 'experiments' in facets and planned_experiments(context.get('question','')): labels['experiments']='Plánované experimenty a vyhodnotenie' - for person in req['people']: - group={'subject':evidence_subject(person,context),'items':{},'missing':[]};seen=set() + for person,work_label in scope_groups(context): + group={'subject':evidence_subject(person,scope_context(context,person,work_label)) if work_label is None else + person+' — '+' '.join(work_label.split()),'items':{},'missing':[]};seen=set() for facet in facets: - rows=[u for u in units if u['person']==person and facet in u['facets']] + rows=[u for u in units if matches_scope(u,person,work_label) and facet in u['facets']] if facet=='progress' and 'results' in facets and any(u['phase']=='progress' for u in rows): rows=[u for u in rows if u['phase']=='progress'] if facet!='assignment':rows.sort(key=lambda u:-unit_score(u['text'],u['kind'],facet)) diff --git a/tools/answer_evaluation/evidence_selection.py b/tools/answer_evaluation/evidence_selection.py index 5edc2ea..2131b84 100644 --- a/tools/answer_evaluation/evidence_selection.py +++ b/tools/answer_evaluation/evidence_selection.py @@ -20,10 +20,10 @@ def identify(row: dict, *, required: bool = False) -> dict: def overview_candidates(context: dict) -> list[dict]: """Keep every main assignment item; do not confuse a backlog with goals.""" - from .content_selection import evidence_units, administrative_only, unit_score + from .content_selection import evidence_units, administrative_only, unit_score, scope_groups, matches_scope rows=evidence_units(context); selected=[] - for person in context['requirements']['people']: - own=[row for row in rows if row['person']==person] + for person,label in scope_groups(context): + own=[row for row in rows if matches_scope(row,person,label)] main=[row for row in own if row['kind']=='assignment'] # Without an explicit assignment, retain a bounded descriptive sample. # A plan or meeting history cannot become the main assignment. @@ -71,11 +71,11 @@ def missing_required_evidence(answer: dict, context: dict) -> list[str]: def overview_units(answer: dict, context: dict) -> list[dict]: """Render all checked assignment items and at most two fallback units per person.""" - from .content_selection import evidence_units + from .content_selection import evidence_units,scope_groups,matches_scope rows=evidence_units(context,answer.get('evidence_quotes',[])) selected=[] - for person in context.get('requirements',{}).get('people',[]): - own=[row for row in rows if row['person']==person] + for person,label in scope_groups(context): + own=[row for row in rows if matches_scope(row,person,label)] main=[row for row in own if row['kind']=='assignment'] for row in main or own[:2]: quote=next(q for q in answer['evidence_quotes'] if q['source_id']==row['source_id'] and diff --git a/tools/answer_evaluation/presentation.py b/tools/answer_evaluation/presentation.py index fd4c34c..ed672d9 100644 --- a/tools/answer_evaluation/presentation.py +++ b/tools/answer_evaluation/presentation.py @@ -108,6 +108,8 @@ GOAL_VERBS = { 'identifikujte': 'Identifikovať', 'preštudujte': 'Preštudovať', 'pripravte': 'Pripraviť', 'vyhodnotte': 'Vyhodnotiť', 'vyhodnnotte': 'Vyhodnotiť', + 'urobte':'Urobiť','naštudujte':'Naštudovať','pozrite':'Pozrieť', + 'prečítajte':'Prečítať','pokračujte':'Pokračovať', } @@ -203,9 +205,10 @@ def no_answer_text(context: dict) -> str: def missing_aspects(facets: list[str], question: str = '') -> str: """Name only the missing requested fields, without empty helper sections.""" - from .question_intent import planned_experiments + from .question_intent import planned_experiments,actual_methods_requested labels={'assignment':'zadaní a cieľoch','plans':'plánovaných úlohách','progress':'priebežnom stave', - 'results':'výsledkoch','problems':'riešených problémoch','methods':'metódach a technológiách', + 'results':'výsledkoch','problems':'riešených problémoch','methods':('skutočne použitých metódach a technológiách' + if actual_methods_requested(question) else 'metódach a technológiách'), 'experiments':('plánovaných experimentoch' if planned_experiments(question) else 'experimentoch a ich vyhodnotení')} values=list(dict.fromkeys(labels.get(f,f) for f in facets)) if not values:return '' @@ -231,7 +234,7 @@ def evidence_subject(person: str, context: dict) -> str: if len(scopes)==1 and next(iter(scopes))[1] is not None: subject+=' ('+str(next(iter(scopes))[1])+')' labels={p.get('work_label') for p in passages if p.get('work_label')} - if req.get('project_number') is not None and len(labels)==1: + if (req.get('project_number') is not None or req.get('exact_work_label')) and len(labels)==1: subject=person+' — '+' '.join(next(iter(labels)).split()) return subject diff --git a/tools/answer_evaluation/question_intent.py b/tools/answer_evaluation/question_intent.py index 5c62fdf..f04c8a9 100644 --- a/tools/answer_evaluation/question_intent.py +++ b/tools/answer_evaluation/question_intent.py @@ -34,6 +34,16 @@ def facet_mode(question: str) -> bool: return any(facet in facets for facet in ['plans','progress','problems','methods','experiments','results']) +def actual_methods_requested(question: str) -> bool: + return bool(re.search(r'\b(?:skutocne|pouziva\w*|pouzil\w*|vyuzival\w*|implementoval\w*)\b|' + r'\b(?:pouzit\w*|aplikovan\w*|vyuzit\w*)\s+(?:metod\w*|technolog\w*|nastroj\w*)\b',normalized(question))) + + +def unresolved_followup(question: str) -> bool: + return bool(re.fullmatch(r'(?:(?:daj|uved|ukaz|vypis|porovnaj|chcem|prosim)\s+)?' + r'(?:obe|oba|obidv\w*|vsetky|prvy|druhy)(?:\s+(?:prace|projekty))?[.!?]?',normalized(question))) + + def output_year_requested(question: str) -> bool: # A supplied year scopes the evidence; it does not itself ask for a year claim. text=normalized(question) @@ -46,7 +56,7 @@ def graph_needs_text(question: str) -> bool: # Explicit graph properties stay on their existing path. if re.search(r'\b(?:tag\w*|klucov\w*|kategor\w*|tem\w*|nazov\w*|autor\w*)\b',text): return False - return bool(set(requested_facets(question)) & {'assignment','plans','problems','experiments','progress','results'}) + return bool(set(requested_facets(question)) & {'assignment','plans','problems','methods','experiments','progress','results'}) def planned_experiments(question: str) -> bool: diff --git a/tools/answer_evaluation/rag_evidence.py b/tools/answer_evaluation/rag_evidence.py index 46e1da8..e8d1518 100644 --- a/tools/answer_evaluation/rag_evidence.py +++ b/tools/answer_evaluation/rag_evidence.py @@ -128,8 +128,9 @@ def requested_types(question: str, people: list[str]) -> dict[str, str]: def project_number(text: str) -> int | None: - match=re.search(r'\bprojekt\w*\s+([1-9]\d?)(?!\d)\b',normalized(text)) - return int(match[1]) if match else None + from tools.retrieval.work_selector import parse_work_selector + numbers=parse_work_selector(text).numbers + return numbers[0] if len(numbers)==1 else None def academic_windows_metadata(body: str) -> list[tuple]: @@ -209,6 +210,8 @@ def descriptive_text(text: str, titles=()) -> str: def academic_content_passages(blocks: list[dict], facts: list[dict], requirements: dict, question: str) -> list[dict]: """Project only retrieved text, respecting person, work type and year together.""" + from tools.retrieval.work_selector import parse_work_selector + selector=parse_work_selector(question) result = []; requested = requirements['people'] profiles = [row['document_path'].removesuffix('README.md') for row in blocks if row.get('title') in requested and PROFILE.fullmatch(row.get('document_path', ''))] @@ -227,7 +230,7 @@ def academic_content_passages(blocks: list[dict], facts: list[dict], requirement continue if requirements['unsupported_year'] or (requirements['years'] and scope[1] not in requirements['years']): continue - if requirements.get('project_number') is not None and project_number(label or '')!=requirements['project_number']: + if not selector.matches(label or ''): continue if not descriptive_text(text, titles): continue @@ -258,9 +261,10 @@ def content_sections(text: str) -> list[dict]: 'zadanie diplomoveho projektu', 'zadanie bakalarskeho projektu', 'navrh na zadanie', 'navrh zadania'}: next_kind = 'assignment' - elif heading and role_value in {'stav', 'priebezny stav', 'vysledky', 'dosiahnute vysledky', 'dosiahnuty stav'}: + elif heading and role_value in {'stav', 'priebezny stav', 'vysledky', 'dosiahnute vysledky', 'dosiahnuty stav','hotove','urobene'}: next_kind = 'progress' - elif heading and role_value in {'ulohy', 'zasobnik uloh', 'dalsie ulohy', 'plan', 'planovane ulohy'}: + elif heading and (role_value in {'ulohy', 'zasobnik uloh', 'dalsie ulohy', 'plan', 'planovane ulohy'} or + re.match(r'^(?:ulohy\s+(?:na|do|pre)\b|do dalsie\w* stretnutia\b)',role_value)): next_kind = 'task' elif heading and role_value in {'napr','napriklad','priklady','priklad'} and not line[:1].isspace(): next_kind = 'other' @@ -269,7 +273,7 @@ def content_sections(text: str) -> list[dict]: if next_kind is not None: flush(); kind = next_kind phase=({'assignment':'assignment','task':'planned','other':'unknown'}.get(kind) or - ('result' if role_value in {'vysledky','dosiahnute vysledky'} else 'progress')) + ('result' if role_value in {'vysledky','dosiahnute vysledky','hotove','urobene'} else 'progress')) else: lines.append(line) flush() @@ -441,7 +445,9 @@ def rag_evidence_context(question: str, payload: dict) -> dict: facts = list({tuple(sorted(row.items())): row for row in facts}.values()) requested = [person for person in people if text_contains_value(question, person, person=True)] requested=list(dict.fromkeys([*payload.get('named_subjects',[]),*requested])) - expected_types = requested_types(question, requested) + from tools.retrieval.work_selector import parse_work_selector + selector=parse_work_selector(question) + expected_types = requested_types(selector.positive_text, requested) query = normalized(question) intent_query = normalized(re.sub(r'[„“"«][^„“”"»]+[“”"»]', ' ', question)) academic = bool(TYPE_PATTERN.search(query) or re.search(r'\b(?:rok\w* prac\w*|nazov\w* prac\w*)\b', query)) @@ -476,6 +482,7 @@ def rag_evidence_context(question: str, payload: dict) -> dict: requirements['years'] = requested_years(question) if academic and not requirements['start_year'] else [] requirements['unsupported_year'] = any(len(value) != 4 for value in stated_years) requirements['project_number'] = project_number(question) + requirements['exact_work_label']=selector.labels[0] if len(selector.labels)==1 else None if requested: facts = [row for row in facts if row['person'] in requested] requirements['available_fields'] = {person:[key for key in ['title','year','start_year'] @@ -489,6 +496,11 @@ def rag_evidence_context(question: str, payload: dict) -> dict: 'aggregation': None, 'answer_candidates': []} if content: context['content_passages'] = academic_content_passages(blocks, facts, requirements, question) + context['work_scopes']=payload.get('work_scopes') or list({ + (p.get('person'),p.get('work_label')):{k:p.get(k) for k in ['person','work_type','year','work_label']} + for p in context['content_passages']}.values()) + requirements['separate_works']=any(sum(s['person']==person for s in context['work_scopes'])>1 for person in requested) + if requirements['separate_works']:requirements['work_scopes']=context['work_scopes'] if requirements.get('facets'): from .content_selection import candidates context['content_candidates'] = candidates(context) diff --git a/tools/answer_evaluation/rag_preparation.py b/tools/answer_evaluation/rag_preparation.py index b629f3e..1ab5a2e 100644 --- a/tools/answer_evaluation/rag_preparation.py +++ b/tools/answer_evaluation/rag_preparation.py @@ -18,7 +18,9 @@ def clarify(payload: dict, reason: str, message: str) -> dict: def prepare_named_content(db_file, question: str, payload: dict) -> dict: - if not content_question(question): + from tools.retrieval.work_selector import parse_work_selector, label_key + selector=parse_work_selector(question) + if not content_question(question) and not selector.specific: return payload if re.search(r'\b(?:s tagom|s klucovym slovom|z kategorie|s autorom dokumentu)\b',normalized(question)): return clarify(payload,'content_filter_requires_graph', @@ -53,9 +55,9 @@ def prepare_named_content(db_file, question: str, payload: dict) -> dict: 'Uveď jeho presné celé meno; ostatné osoby nebudem používať ako náhradu.') if not named: return payload - types=requested_types(question,[r['title'] for r in named]) + types=requested_types(selector.positive_text,[r['title'] for r in named]) years=requested_years(question) - selected=[] + selected=[];scopes=[] for person in named: rows=list(conn.execute('SELECT text,heading_paths_json FROM chunks WHERE document_path=? ' 'AND COALESCE(published,1)=1 ORDER BY chunk_index',(person['path'],))) @@ -68,15 +70,22 @@ def prepare_named_content(db_file, question: str, payload: dict) -> dict: if not scope or (types.get(person['title']) and scope[0]!=types[person['title']]) or (years and scope[1] not in years): continue label=label or WORK_LABELS[scope[0]]+(f' {scope[1]}' if scope[1] is not None else '') - if project_number(question) is not None and project_number(label)!=project_number(question): + if not selector.matches(label): continue records[(scope,normalized(label))]=label if (scope,normalized(label),text) not in seen: seen.add((scope,normalized(label),text));frames.append('Sekcia: '+label+'\n'+text) - if len(records)>1: + if selector.multiple and (any(label not in {label_key(v) for v in records.values()} for label in selector.labels) or + any(not any(project_number(v)==number for v in records.values()) for number in selector.numbers)): + return clarify(payload,'unresolved_work_selection', + 'Niektoré z požadovaných označení prác osoby '+person['title']+' sa v indexe nenašli. ' + 'Uveď presné nadpisy zo zdroja; chýbajúcu prácu nevynechám z porovnania potichu.') + if len(records)>1 and not selector.multiple: options='; '.join(dict.fromkeys(records.values())) return clarify(payload,'ambiguous_work',f'Ktorú prácu osoby {person["title"]} máš na mysli? ' f'V indexe sú tieto možnosti: {options}. Uveď typ práce a rok alebo presné označenie projektu.') + scopes.extend({'person':person['title'],'work_type':scope[0],'year':scope[1],'work_label':label} + for (scope,_key),label in records.items()) from .content_selection import prioritize_frames frames=prioritize_frames(frames,question) source=next((s for s in payload.get('sources',[]) if s.get('document_path')==person['path']),None) @@ -92,4 +101,4 @@ def prepare_named_content(db_file, question: str, payload: dict) -> dict: selected.append(source) selected=[{**source,'source_id':f'S{i}'} for i,source in enumerate(selected,1)] return {**payload,'sources':selected,'source_count':len(selected),'context':build_context_text(selected), - 'named_subjects':[p['title'] for p in named]} + 'named_subjects':[p['title'] for p in named],'work_scopes':scopes} diff --git a/tools/answer_evaluation/verified_rag.py b/tools/answer_evaluation/verified_rag.py index 8ea7a8c..7be9861 100644 --- a/tools/answer_evaluation/verified_rag.py +++ b/tools/answer_evaluation/verified_rag.py @@ -13,7 +13,8 @@ from .rag_evidence import descriptive_text, rag_evidence_context, quote_sections from .answer_coverage import answer_coverage, coverage_notices from .presentation import (content_lines, fact_sentences, no_answer_text, work_subject, clean_answer, evidence_subject, comparison_table) -from .content_selection import facet_lines, evidence_units, missing_facets, fit_candidates, has_qualification +from .content_selection import (facet_lines, evidence_units, missing_facets, missing_work_facets, + matches_scope, fit_candidates, has_qualification) from .evidence_selection import (resolve_selection, EvidenceSelectionError, missing_required_evidence, overview_units) @@ -294,10 +295,10 @@ def rag_answer_issues(answer: dict, context: dict) -> list[str]: if kind not in kinds: issues.append('missing_' + kind + '_evidence') if requirements.get('facets'): - available={(c['person'],facet) for c in context.get('content_candidates',[]) for facet in c['facets']} + available=context.get('content_candidates',[]) if answer['status']=='ok': - for person,facet in missing_facets(answer,context): - if (person,facet) in available: + for person,label,facet in missing_work_facets(answer,context): + if any(matches_scope(c,person,label) and facet in c['facets'] for c in available): issues.append('missing_'+('progress' if facet=='results' else facet)+'_evidence') elif answer['status']=='no_answer' and available: issues.append('rejected_supported_content_comparison') @@ -430,17 +431,18 @@ def render_rag_answer(answer: dict, context: dict) -> str: quote = selection['evidence'] passage = next((p for p in context.get('content_passages', []) if quote_matches_passage(quote,p)), {}) - identity = (passage.get('person'), passage.get('work_type'), passage.get('year')) + identity = (passage.get('person'), passage.get('work_type'), passage.get('year'),passage.get('work_label')) groups.setdefault(identity, []).append(selection['text']) table = None if style == 'overview' and len(groups)>1 and re.search(r'\bporovnaj\w*\b',normalized(context.get('question',''))): from .presentation import goal_text - table=comparison_table(['Oblasť',*(evidence_subject(person,context) for person,_,_ in groups)], + table=comparison_table(['Oblasť',*(person+' — '+label if req.get('separate_works') and label else + evidence_subject(person,context) for person,_,_,label in groups)], [('Zameranie podľa podkladov',['\n'.join(goal_text(text)[0] for text in units) for units in groups.values()])]) if table is not None: lines.extend(table) - for (person, kind, _year), selections in ([] if table is not None else groups.items()): + for (person, kind, _year,label), selections in ([] if table is not None else groups.items()): if style == 'overview': units = selections else: @@ -451,7 +453,9 @@ def render_rag_answer(answer: dict, context: dict) -> str: units.extend(block.strip() for block in re.split( r'(?=^[ \t]*(?:\d+[.)]|[-+*])\s+)', text, flags=re.M) if block.strip()) - lines.extend(content_lines(units, person, kind, context.get('question', ''))) + rendered=content_lines(units, person, kind, context.get('question', '')) + if req.get('separate_works') and label:rendered=['**'+person+' — '+' '.join(label.split())+'**','',*rendered[1:]] + lines.extend(rendered) else: sources = {row['source_id']: row for row in context['source_blocks']} lines += ['Zo zdroja ' + str(sources[quote['source_id']].get('title') or quote['source_id']) + @@ -527,6 +531,9 @@ def generate_verified_rag(question: str, payload: dict, complete) -> dict: + ('claims=[] a source_urls=[]; citácie dosadí aplikácia podľa IDs.' if context['requirements']['response_mode']=='quotes' else 'Pri mixed vyplň iba výslovne požadované claims a cituj aj ich zdroje.')) + if context['requirements'].get('separate_works'): + selection_policy+=(' Rovnaká osoba, typ a rok nezlučujú rôzne work_label. ' + 'Vyber dôkazy pre každú dostupnú požadovanú časť každej práce zvlášť; chýbajúce časti označí aplikácia.') facets=context['requirements'].get('facets',[]) if facets: support={f'osoba_{i}':[f for f in facets if any(row.get('person')==person and f in row.get('facets',[]) diff --git a/tools/answers/answer_delivery.py b/tools/answers/answer_delivery.py index d2eb048..e596947 100644 --- a/tools/answers/answer_delivery.py +++ b/tools/answers/answer_delivery.py @@ -33,6 +33,11 @@ def deliver_answer(question: str, *, engine: str = 'rag', limit: int = 5, 'accepted_for_delivery': False, 'answer': 'Agent práve spracúva iné odpovede. Skúste to o chvíľu.'} try: model = complete or complete_with_configured_model + from tools.answer_evaluation.question_intent import unresolved_followup + if unresolved_followup(question): + return {'status':'ambiguous','reason':'missing_followup_context','accepted_for_delivery':True, + 'answer':'Uveď osobu a presné označenia prác alebo úplnú otázku. Samotné „obe“ bez predchádzajúcich možností neviem spoľahlivo vyhľadať.', + 'claims':[],'source_urls':[],'engine':engine,'index_version':None,'complete':False,'truncated':False,'truncation_reasons':[]} unsupported=unsupported_constraint(question) if unsupported: return {'status':'ambiguous','reason':'unsupported_constraint','accepted_for_delivery':True, @@ -51,7 +56,8 @@ def deliver_answer(question: str, *, engine: str = 'rag', limit: int = 5, context = retrieved elif engine == 'graphrag': from tools.answer_evaluation.question_intent import graph_needs_text - if graph_needs_text(question): + from tools.retrieval.work_selector import parse_work_selector + if graph_needs_text(question) or parse_work_selector(question).specific: return {'status':'ambiguous','reason':'text_evidence_required','accepted_for_delivery':True, 'answer':'Táto otázka vyžaduje textové podklady o zadaní, priebehu alebo výsledkoch práce. ' 'Použi /rag s rovnakou otázkou; graf samotný tieto výňatky neposkytuje.', diff --git a/tools/graphrag/evaluation/reproducibility.py b/tools/graphrag/evaluation/reproducibility.py index d7372c7..71f853d 100644 --- a/tools/graphrag/evaluation/reproducibility.py +++ b/tools/graphrag/evaluation/reproducibility.py @@ -48,6 +48,7 @@ def content_fingerprints(questions: Path, root: Path = ROOT) -> dict[str, Any]: 'tools/answer_evaluation/rag_evidence.py', 'tools/answer_evaluation/verified_rag.py', 'tools/rag/llm_context.py', 'tools/answer_evaluation/question_intent.py', 'tools/answer_evaluation/content_selection.py', 'tools/answer_evaluation/evidence_selection.py', + 'tools/retrieval/work_selector.py', 'tools/answer_evaluation/rag_preparation.py', 'tools/answer_evaluation/answer_coverage.py', 'tools/answer_evaluation/graph_answer_prompt.txt', 'tools/rag/evaluation/rag_metrics.py', 'tools/rag/evaluation/rag_runner.py']) diff --git a/tools/rag/llm_context.py b/tools/rag/llm_context.py index fe5b2a1..807b384 100644 --- a/tools/rag/llm_context.py +++ b/tools/rag/llm_context.py @@ -12,7 +12,7 @@ from tools.retrieval.text_budget import split_to_fit SOURCE_FIELDS = ('source_id', 'title', 'author', 'document_path', 'source_url', 'published', 'categories', 'tags', 'section') FIELDS = ('status', 'reason', 'query', 'engine', 'strategies', 'instructions', 'answer_format', - 'complete', 'truncated', 'truncation_reasons', 'index_version', 'source_count', 'clarification', 'named_subjects') + 'complete', 'truncated', 'truncation_reasons', 'index_version', 'source_count', 'clarification', 'named_subjects', 'work_scopes') def compact_rag_payload(data: dict, *, max_tokens: int | None = None, diff --git a/tools/retrieval/question_constraints.py b/tools/retrieval/question_constraints.py index 66fe859..89846cd 100644 --- a/tools/retrieval/question_constraints.py +++ b/tools/retrieval/question_constraints.py @@ -1,17 +1,18 @@ """Fail closed for recognized query operators without an implemented meaning.""" import re -import unicodedata def requested_years(question: str) -> list[int]: """An unquoted four-digit year remains a condition without 'v roku'.""" - text=re.sub(r'[„“"«][^„“”"»]+[“”"»]', ' ', question) - return list(dict.fromkeys(int(value) for value in re.findall(r'\b\d{4}\b',text))) + from .work_selector import selector_years + return selector_years(question) def unsupported_constraint(question: str) -> str | None: - text = ''.join(c for c in unicodedata.normalize('NFKD', question).casefold() - if not unicodedata.combining(c)) + from .work_selector import parse_work_selector + selection=parse_work_selector(question) + if selection.unsupported_negation:return 'exclusion' + text=selection.positive_text # Do not treat words inside quoted academic titles as query operators. text = re.sub(r'[„“"«][^„“”"»]+[“”"»]', ' ', text) academic = re.search(r'\b(?:prac\w*|projekt\w*)\b',text) diff --git a/tools/retrieval/work_selector.py b/tools/retrieval/work_selector.py new file mode 100644 index 0000000..694114f --- /dev/null +++ b/tools/retrieval/work_selector.py @@ -0,0 +1,79 @@ +"""Source-heading selectors; negated project numbers never become positive filters.""" +from dataclasses import dataclass +import re +import unicodedata + + +def normalize(text): + return ''.join(c for c in unicodedata.normalize('NFKD',text).casefold() if not unicodedata.combining(c)) + + +def label_key(text): + return ' '.join(re.sub(r'[^\w]+',' ',normalize(text)).split()) + + +QUOTED=re.compile(r'[„“"«]([^„“”"»]+)[“”"»]') +HEADING=re.compile(r'(?:diplomovy|bakalarsky|timovy|vyskumny|vedecky) projekt(?: [1-9]\d?)? [12]\d{3}|' + r'(?:diplomova|bakalarska|dizertacna) praca [12]\d{3}') +NUMBER=re.compile(r'\bprojekt\w*\s+([1-9]\d?)(?!\d)\b') +NEGATIVE=re.compile(r'\b(?:nie|okrem|bez|s vynimkou)\s+(?:(?:diplomov|bakalarsk|timov|vyskumn|vedeck)\w*\s+)?$') + + +@dataclass(frozen=True) +class WorkSelector: + labels: tuple=() + excluded_labels: tuple=() + numbers: tuple=() + excluded_numbers: tuple=() + unnumbered: bool=False + multiple: bool=False + positive_text: str='' + unsupported_negation: bool=False + + @property + def specific(self): + return bool(self.labels or self.excluded_labels or self.numbers or self.excluded_numbers or self.unnumbered) + + def matches(self,label): + key=label_key(label);match=NUMBER.search(key);number=int(match[1]) if match else None + return ((not self.labels or key in self.labels) and key not in self.excluded_labels and + (not self.numbers or number in self.numbers) and number not in self.excluded_numbers and + (not self.unnumbered or number is None)) + + +def parse_work_selector(question): + text=normalize(question);positive=[];negative=[];excluded_spans=[] + for match in QUOTED.finditer(text): + key=label_key(match[1]) + if not HEADING.fullmatch(key):continue + neg=NEGATIVE.search(text[:match.start()]) + if neg: + negative.append(key);excluded_spans.append((neg.start(),match.end())) + else:positive.append(key) + numbers=[];excluded=[];unsupported=False + for match in NUMBER.finditer(text): + if any(quote.start()<=match.start()1 or len(set(numbers))>1 + return WorkSelector(tuple(dict.fromkeys(positive)),tuple(dict.fromkeys(negative)), + tuple(dict.fromkeys(numbers)),tuple(dict.fromkeys(excluded)), + bool(re.search(r'\bnecislo\w*\b|\bprojekt\w*\s+bez cisla\b',text)),multiple,cleaned,unsupported) + + +def selector_years(question): + selector=parse_work_selector(question) + text=QUOTED.sub(' ',selector.positive_text) + years=[int(v) for v in re.findall(r'\b\d{4}\b',text)] + years.extend(int(label.split()[-1]) for label in selector.labels) + return list(dict.fromkeys(years))