From 24a87dc4b04e8e2019b8d99e9f8c52ec4f98453d Mon Sep 17 00:00:00 2001 From: jp170na Date: Sat, 7 Nov 2026 22:02:33 +0100 Subject: [PATCH] Publish content and retrieval as one versioned snapshot; sync fork daily --- .dockerignore | 4 +- README.md | 13 +- app/main.py | 4 +- app/routes.py | 21 ++ deployment/docker-compose.yml | 3 + packages/claude/client.py | 2 + packages/claude/dist/packages.json | 4 +- packages/claude/dist/zpwiki-graphrag.zip | Bin 8852 -> 8892 bytes packages/claude/dist/zpwiki-rag.zip | Bin 9084 -> 9125 bytes test/test_answer_health.py | 54 ++++ test/test_content_publication.py | 145 ++++++++++ test/test_git_runner.py | 13 + test/test_knowledge.py | 30 +- test/test_publication_health.py | 13 + test/test_sync_jobs.py | 17 ++ test/test_wiki_git.py | 17 ++ tools/answers/model_client.py | 99 +++++-- tools/claude/build_claude_packages.py | 2 +- tools/git_runner/sync_fork.sh | 20 +- tools/knowledge/build.py | 47 +++- tools/knowledge/publication.py | 119 ++++++++ tools/knowledge/query.py | 5 +- tools/knowledge/store.py | 336 ++++++++++++----------- tools/rag/rebuild_index.py | 7 - tools/runtime/answer_health.py | 83 ++++++ tools/sync/sync_indexes.py | 83 ++++-- tools/sync/sync_jobs.py | 102 ++++--- tools/sync/wiki_git.py | 10 + 28 files changed, 966 insertions(+), 287 deletions(-) create mode 100644 test/test_answer_health.py create mode 100644 test/test_content_publication.py create mode 100644 tools/knowledge/publication.py create mode 100644 tools/runtime/answer_health.py diff --git a/.dockerignore b/.dockerignore index e64104f..4254a13 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,7 +11,9 @@ htmlcov/ data/ docs/ integrations/ -tools/git_runner/ +tools/git_runner/* +!tools/git_runner/sync_fork.sh +!tools/git_runner/ssh_known_hosts **/*.md experiments/ test/ diff --git a/README.md b/README.md index d5ad5e2..520f8d4 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,12 @@ odpovede o prácach, projektoch, rokoch a súvisiacich vzťahoch so zdrojovými 5. Backend skontroluje hodnoty, väzby, citácie a pokrytie požadovaných údajov. 6. Z overených podkladov zostaví odpoveď; chýbajúce informácie výslovne označí. -Aktualizáciu možno spustiť manuálne alebo webhookom z Gitea. Oba indexy -sa obnovujú z rovnakého snapshotu a používajú spoločnú verziu dát. +Aktualizáciu možno spustiť manuálne alebo webhookom z Gitea. Docker nasadenie +ju plánuje aj každých 24 hodín (`WIKI_SYNC_INTERVAL_SECONDS=86400`). +`SYNC_UPSTREAM_FORK=1` bezpečne dopĺňa fork `jp170na/zpwiki` z `KEMT/zpwiki`; +do originálu nezapisuje. Vyžaduje bežiaci backend a SSH prístup do forku. +Textový index, metadátový graf a už zapnutá obsahová extrakcia sa pripravujú +z rovnakého snapshotu a aktivujú spoločne. Pri chybe zostáva predchádzajúca verzia. ## Vyhľadávanie @@ -62,7 +66,10 @@ v Docker sieti. Dočasná verejná URL vyžaduje zodpovedajúce OAuth nastavenie RAG, GraphRAG, kontrolovaná cesta odpovedí a pripojenie cez Claude sú funkčné. Systém obsahuje embeddingovú cache, kontroly dostupnosti a persistentnú -synchronizačnú frontu. Historické evaluácie sú oddelené od prevádzkového kódu. +synchronizačnú frontu. `/health` samostatne uvádza vyhľadávanie, obsahové +poznatky a generovanie. Dostupnosť modelu eviduje po jeho použití alebo po +explicitnom autentifikovanom `POST /health/generation`; samotná kontrola +`GET /health` model nevolá. Historické evaluácie sú oddelené od prevádzkového kódu. Samostatný hlavný agent, CRZP a spracovanie plných textov prác sú ďalšie etapy. ## Organizácia projektu diff --git a/app/main.py b/app/main.py index 1b96d00..92758e3 100644 --- a/app/main.py +++ b/app/main.py @@ -2,6 +2,7 @@ from __future__ import annotations import asyncio import logging +import os import sys from contextlib import asynccontextmanager from pathlib import Path @@ -68,7 +69,8 @@ async def lifespan( application.state.embedding_ready = True return result - application.state.sync_jobs = SyncJobs(DATA_DIR, runner=run_sync) + application.state.sync_jobs = SyncJobs(DATA_DIR, runner=run_sync, + periodic_interval_seconds=float(os.getenv("WIKI_SYNC_INTERVAL_SECONDS", "0"))) application.state.sync_jobs.start() try: yield diff --git a/app/routes.py b/app/routes.py index d490007..78d4e63 100644 --- a/app/routes.py +++ b/app/routes.py @@ -209,6 +209,8 @@ class HealthResponse(BaseModel): security_configured: bool index_version: dict[str, Any] | None = None tools: dict[str, Any] + capabilities: dict[str, Any] + answer_ready: dict[str, bool] class RagSource(RetrievalSource): @@ -444,12 +446,23 @@ def health(request: Request) -> dict[str, Any]: embedding_ready=getattr(request.app.state, "embedding_ready", False), ) ready = tools["rag"]["ready"] and tools["graphrag"]["ready"] + from tools.runtime.answer_health import answer_capabilities + capabilities = answer_capabilities(DB_FILE, tools, version) + content = capabilities["knowledge"] + generation = capabilities["generation"]["ready"] + answer_ready = { + "rag": tools["rag"]["ready"] and content["ready"] and generation, + "graphrag": tools["graphrag"]["ready"] and content["ready"] + and content.get("graph", {}).get("ready", False) and generation, + } return { "status": "ok", "ready": ready, "index_version": version, "tools": tools, + "capabilities": capabilities, + "answer_ready": answer_ready, "database_exists": (database_exists), "search_engine": ("hybrid_fts5_embeddings"), "rag_enabled": True, @@ -458,6 +471,14 @@ def health(request: Request) -> dict[str, Any]: } +@router.post("/health/generation", include_in_schema=False, + dependencies=[Depends(require_search_api_key)]) +def probe_answer_generation(): + """Explicit authenticated model probe; GET /health never incurs model calls.""" + from tools.answers.model_client import probe_generation + return probe_generation() + + @router.post( "/rag", operation_id=("retrieve_zpwiki_context"), diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml index 000f8d5..dfbdd8c 100644 --- a/deployment/docker-compose.yml +++ b/deployment/docker-compose.yml @@ -19,6 +19,9 @@ services: environment: ZPWIKI_ROOT: /zpwiki + WIKI_SYNC_INTERVAL_SECONDS: ${WIKI_SYNC_INTERVAL_SECONDS:-86400} + SYNC_UPSTREAM_FORK: ${SYNC_UPSTREAM_FORK:-1} + ZPWIKI_SYNC_KEY_FILE: /run/secrets/zpwiki_ssh_key CHUNK_MAX_TOKENS: "450" CHUNK_OVERLAP_TOKENS: "70" CHUNK_MIN_TOKENS: "80" diff --git a/packages/claude/client.py b/packages/claude/client.py index a50366f..2f6362e 100644 --- a/packages/claude/client.py +++ b/packages/claude/client.py @@ -336,6 +336,8 @@ def backend_health(config=None): "status": "ok" if value["ready"] else "error", "ready": value["ready"], "index_version": value.get("index_version"), + **({"capabilities": value["capabilities"]} if isinstance(value.get("capabilities"), dict) else {}), + **({"answer_ready": value["answer_ready"]} if isinstance(value.get("answer_ready"), dict) else {}), "tools": { engine: { k: v diff --git a/packages/claude/dist/packages.json b/packages/claude/dist/packages.json index 5c56b4d..6b33982 100644 --- a/packages/claude/dist/packages.json +++ b/packages/claude/dist/packages.json @@ -3,12 +3,12 @@ { "archive": "zpwiki-rag.zip", "name": "rag", - "sha256": "c2b0902370aaa43a972edf0efa38a1211fe97575c4a8a68b0eb7cea1bbec5ccf" + "sha256": "c7a207660004c1e4e7ae7d507006df231c666d13c53067bd5896a022ef44b64d" }, { "archive": "zpwiki-graphrag.zip", "name": "graphrag", - "sha256": "e127fb2c97b5ed8279904c754ae394a8fc3be9fa210a6f7b13fff9499d8888a6" + "sha256": "1f340a0298625b7dd5f427e8501dde8ec781cc8d72e7a4f50b9ad4b21de6b1fa" } ], "schema_version": 1 diff --git a/packages/claude/dist/zpwiki-graphrag.zip b/packages/claude/dist/zpwiki-graphrag.zip index 77eb8db0fd6b3a55f0bcd5263999731e0a7c6654..ea83b5bda446ba7863a942de82b91867cb13e8ea 100644 GIT binary patch delta 5176 zcmV-86vyk7MZ86@f(!-oR?gu+lZOl=e@jloKn#ZWJw=o?3ggT(+yK!X2Z$VxrybFx z=_CPFRqxI;1%VJt|G)kDb=ed}*&)60;M>HfB_{UeQ!%H7A8wxKfXiWS`=jTqm+zcn zi0oS&rVmc#n#98{4y(OOIPMefzV$hs$73AkoA>(iyxms4zk4Abc*^o62gNTleXo}?Pj zH@ImGF&Ji)iaG%B+NsX!5LDB2XoXSE3jl;{n!LTzL@1Q@j)Z{jFZ{G{I)Aa1k;(}2 zjigXx=7Kgp7({KMqtLjzv8)l$Th=;{4_?%r$lZYf4LDI|^mV;XABWg4eLJ4#{c*Ul zWd--e$^rX-ZavYSdu;Q6rnpwXD*wbKRIZZc=DPU=P)h>@6aWAK2mk;8Apq2c+Gb@E z0037%000^QlN%x&f4w`)mfOg6`~Hdwl9NNq6xGtq*eA4UTUL6;if7b))Ga5Tb_b^j z6v=9WC>RB>ADaGzIsOJMvY3T;S?L_lhMND%z4ZX9fFjjWlwi>f)Z^Bz`@Xk|Q52o7 zX_k#h$_kdJEKhD8&P&Efl5v(-L@sH`QZlEf?WCs3 z&fo=2(~`9fmr~^RT{Nww0a?&$nemwgcLJYwhy-NpdtGKQYRt+~l$|z>pWjxpztl`t z2M6Cg{g3$Nf6?j5@r%==_;-IgJ31v(@`OAfj}M=GZvGt{z4+?+3)M4-O1g+fBwCcT zSmOKe;PmY2*{f64{W=mW?9D|?b9u!|{GjU@U(}+O_@NMt?hXzP6j6X7$e|-_9UNUJ ztU&S(CkF%`;KdnmXAQKX3r09Fj1=`O;|WlQ0@X+&e^NFgIoq%jxI>r)KoseawN(j- z;LbH?&8jcNOP2DICDp(u$oThXXD7Dr_aGk`D-AX&o5QlYCpN6a=;;HfY5In0T}?&w=AEYl{Fh$@`!xS7kVy0nE=mA z^|4|%f9*%b*G$yagv>KRE8xRthli>IltNs^%c?3SWF|xgU0>1+q(SxmR;{sS)l#I* z76C#!NHXalyrCIv7TJ5(w9R?Oa=K>FD&%;y z90QuX7z~@aeBMlN7g75tZ~O*)Pyt{S6#Rjme>NPm>jJn_OIA97odp0Xjoy?*g^>I@ zQreCxGd_)nNAZli4Uy9EuB8uDO(G z8n8b%aKRk^3*fr!t#UCTRrA7sWI*luK4fq821l9+Xfaid6p0I(214fT0)o>ra ze}BY*8eJc*))#j_eJ~0{Xx8BPR81K<@B?TBNXBzOou{!TU0o`Z8V$A5Dz&hadjg@S z0PR3R2fejIlKAT7*C2+PR@gg;W;9uWmxw{1w174eIxh7~^e7sR?gjnn$@BORM}O+9 zKnos&f_9+yAu4qpgy5b%|K{lU)maCwf6?<;GN40>_7oI-IJ_^eUw?i4?=c!UqnBVC zP<;r`!Eisu#tKYNbmxN`1{g3_45St72$V5xdQj721*Vmg5jndl?CaAC=rgOUpa^}G z_%z0>K@;t?1Gy(xAPXQ=uEQL|hb%I#Z9IPYTLpdW3m}Xd%Vf#G@F-f7;@I z8Qt|+RjRQ#btzA^Hw(QZAwEb?)eGtj7@U&Aj&bI0R< zj7Knl6)3P^xXtL#Wz?{=Y0G-0d4|L74bByMJ1cF5>g8m*E!u#p3zyxq6vw%UBgwJ3OC6!oLf{1iQZ^4SN zdb53#-qQ6rYRz&47}OB8mHd{ah{8#^{hO@n0>;lE%7aeq6C0Xu|2`(CE0I^F$m&Z{ zlzf?O-$<5oMxdFkU?hdOfBKRw1vV&>ZQmrk_7()ip79hiE4>=T)j)7TasUO8FJayF z_RUXkNFl0{&GPLJL=*sKljA}jB+DLUa}e4w(Cc)wlBHVOe{uoBFQ6bRrU5w)?(r<#qtG!@fJe<06ziZ?M6Td3+OhaQ~SP! z5WK$Jz8#YjQUCNKZ@Yma?|!7@5m;de=YhTCkl{2dfO(XBIk3asgzPQPw&2`vdjNl6n3S7LBcdqw9kaoLR>22B8eFR zR(Frs{otCX5qbALCo={LK#9yi5oR}_GgY0l%v&7d%#t%OB;DnmT_&KkJWtv6|61z7 zlc2ea6||nzf6XLaf{Oh>Ha>*|s~V|OLaudL8q14T3+=a~5RSAqZsQD$D{b@%GR2av zz*GZZ<`^aDzJ*4>r23@~@iGM;ghqN(rpToc0pCLu+SJj!oUv;T_@F^DgwS!TiSHe+ zi0N}Jkr2AF<1q<+Ds<@=SE&;8@GOnNt;ArL_(uBze^s04DIauLkF3%l1Mb5V9!$7g zp*L|eyGV+d>%S6YVhP?U1X451u%vLDo?03jXV0#3F$4k>)x5D z#1xidtPKow2s1HNpb-IWDg@6fRrwi_nMiM@h?|L9!nv7&*rqqO)b-NTuo?`_e9VP@ z)~wZTe*gvbUqxIE${u}Df~!ueEfc{qkB3ET~4mUD%qn}ZMrLpl^SwX$g| zSBhXyO^l->I1sVlFYr+7B_l z49>aRy@%I#jwqR=S(Dt$A+C6}?2>*~^DK=;f5CD{z@1{`a5%e8ctjTgZg*KaSdE9>W@@t!VX)!Zjrsy~934u-crf!K6M)&}RDsgTH zc=Q5F(5fo+EbBf-3elktrj_}9+VO-8Fn1^b=!P4)Z#M*doHP_?3n`ytwIf_j;+1`_ATO!)M z26AodlHxV682yOefPwFo(r}}%TNgWWed;LaL`PkReTTBI7C`?X9SVT(>2_64e{B{S z28(3sbP)W4#`U}%e$kA_GZhzjJKPj(8`Ot8=NRbi&#yE^sl-3g3dWCxz3AOKUe`l(1u*B+r z@8A*?aE-Esq10FvcLrtjpO3&;FTS%ync?6!(?84a{%HvGFu1|s=bkY)aC|J=_VdRV z&DqR9Os)LvD4}}W>Xx4T;;y&3bF%(irev4f-N?BYz28$Vg~K% z|DC(@-YTeMS~aj0Z$9`Wz_rl6Bw3xZIOTMa3t905j9qi;z6X8?m|joK?`#aW9GanS zN5~~))@U%)OuS9d3p4^*W}98=#PYq-hc?G1{~sD3(OP}eabeAbf4wDgZ@~#zS_h-< zDo9ElG5ZVrBY6qAkFI_~+*i~U+Wdw&%2n8MCP6bv)-@!{h%}Q4E!en|i<$JrA%O{0 zeO;4u&aAuZWjpu;3by@Y+}-8J-D8{L?Cv4&Su|W-3k=*{7z;uTouBvv@7^OEeH*AK z+-n8_am3XYjFg0Ge}*eA;sKV!rXCbOM=-S0Lbk^cg{kKOAG!(|42+ur+Q;`K$O?X^ z_+fJsT4^b!~7VYTtJ z4Su|7jPQqg;{JJ47!Pw{Bv=q_A|}eWM)`UNCZAdw8xxOJeY#*#vxojdc8v8u=e@}n zAye3AIzFB4TM%wa6d;-%X84wfqPr&#eoh9_oF)|}2RL)X*_FBzDkWd!SgDUx7$(*K zMAXo5-?dHTf3bgYSr>SJL>CVVT2`E@TVHj)%EeW#FRM0HLRErnD?z&u!D}zUzCCnR zNUzw(qWGr2<#!oeRuqU1G&${^ZF2lIs?}AZWf2eq@I&4zh?{O_tD5%oLCm3#_ zdW6;rx{p2Kq%{}&qA$JeF-_A!fX+^y*2_iqCk*Du&FaA?Hm98&ZKv$mikd2o{2v7x z4ld1gd4xbBx}1NZuEV zk}dT4e_A0S`a%S^+F(Q=R)IIboiv!&P$+ zUdD5t!$?C;y-H#abuqTR^Hyq#h@oRv_13?)f8INeGDPw&`AL1YG_)n#=D3Z~tXdBG z_1EAbO`vwtD`J6XeWal}no|jpNxyWUn*73sZyUM?3V3pRv+FPcpT2!iYlNb@U4cw! zzFJ+&&~R6TXsf(x!nq!NaZj}InYSgj9hK*gsqpSGWkE%9ZnTJ17J_Cx;}uu8R_w`?+j?==DSx>B&K+nO8Bnuky(XdbHs-bN zThAN#p7o3}OaeZ|o}l!5@L2&~!473%M`VNIDgW{G7P~uf3bi$0AJbRfu27x40`x$_0k+d{eiKfJ$-_I zl08c3hS5smDGs>hUpnVa70cKfzM3mD=D14D8|xouJcBGA53=SJba+Q+O-W3u-|yM- zlr`4sG^>Oa$K=a8(?-PB&H#rsICDvw6b?a@kkSI>%@Kl%zu>d20k@dzQtu1(e`~^I zN%IA!9x%AU!MJgbSDz_SfjSZM2FDzK3t(xc{s*de{mNaxYS+sutRHt;T;KiPt9AW5 zb-hb)-F#$c)i96@gsupxKu7z`CeqQLKz&WfJ+Y3KR^+DfI<8w*zt1Uws`kgXKUX=CM8$D1f9llDv~%Je>yWUr-`YVVrz{fl0%c6M!zv0q7C4=RL9G$_65N_ zd}x^j!Xr3@z*D2`_`Bpeg$++5{fpT>Ffa;oAe@H|S1%lZwn(=Rrat8wDepX02RwHA z7klTK2xdg63U8+%x}s#U`jMEPh!g`pnEC1-LXgeXLlgG4K}IL$@v72UUu0z zTUW!((W>DPT1ad)QLa8>S1oIGvr3<_s|I}rt>8C&Rz|8_T-k_bcDRDwY;C_qHS{fk z>I3KMq>0r7mnR_+Wk5#?l$O&$G(yHtAaWYI%JmqLHag_h!kl9;O}RtRBp4gV_=oNY4lz zdmPzLF$i+^Xq*%ciZ1yc`SFQ*Jsgisr1VQB?u$Hcxd=Dkj%(TE;qI{vv|ZMIys}ii zd6ANbEZozy{Fc(}O){PGwA-gb^SRLJkG_`Me9hB(_kRBP{`Az2{NWAxB1@HDe`|64 zV%A1N>rgBkCpryiZLkT*aiXX@%K;prSLo2VkbH1pkbr$kMrjL9?F%Y%NhQzg@|Z5U zCTZu}9o_-Nm@rx%NO@%KZP!JLLfUtM5UddlA}R6U5hF)&Vx5YH#cDSH!k;%Ow@KC^543F7F*pM-VX0jO9KQH000080000X056gX zjXDwl0Czr<{|zF4U6b3$aqs&pCP*qp$`rNI-Psk|v@I)pXT^8gE$zyQZ@obcfgy<{ zh=DNx_d~Zo;mW^34|%u;zvW3)z8C8LSGs#;m)A)6?JGGYo>@bj{Or z#A1<)EEZXG^Kf1W!J;$~S;^Fr7b0e}8}=;aRVE(QDM*$JB!NxTr_ zAP5c)=7n6dFq~IqRfsTT$vT%s3Bxm4@-mT`IykUx#Ukg05_(R|OCF_MDWUA7y2;K+ zavsNpXc{h+%;Zdx_~di*@8IagSI=Ki&mbuHA{en?QSf|; z@56)Bv!`dTPO1BKAXnI%$&hF2N)-6P*Ry0%$x7h|AsE~p92^i)fFa1CBWxWUT}L8E z@(w2l3?AUc8E|I}wBifF5?~n1t67>vKphTLW08!1#fW8MBMRUS6BYm?(jjZh0uaHS zYtEWgU&xmtP6`o~gEm3Nzdt)Wv3`A9`!u!+SSmeoC>89q=VosXdTFC#Y3M+Tp@ zT8!9I@K_WIvAcpb3=n(@0b-Q_4w(Riep?I30Fb^F+4QWa#L$vQX7gmB=K_=o@Vuap zmAGktKFVY*WK~YsJe9lzK74j~NFAUQ@+w@GWj09HZ4AJ}Ps!?C!|fjgBFg#*}G0Fct?O<9mjm0Sm; z?Wi*2)A0E>$NzdXK*(ck86b|~8kjd^|I7~M;V*vC$AehM6D3+Q8$`sRFc-x-Q3`DX z2?vNtc=+?f!=R5{O-@w&G16OUR6wZXuoTyTk9SXCeLxA43{cJ@G0%cXCdLAkPbcoTOf}#(H_r>+=uaEydL<48^5^Mvi z4@)vI+>f!b1k)4TwZRQN3>Xswc_}&qWlWnMRQFhcY2;+Y&Tex1`m_Z4%&O8WLLVhQ zwK1#FL_6(3?#UI%0tl7sFvswr@-)#l9>3aK1%2!bAdDKTXeq$(kg_6S(fT5PZSlVf z?)t1MXe>@$N>!rO@{PfMG@DNy!;xd@9XE; z%^(_s-OD)`BwzuwMYwH0&32lHofe9A0^YkzdWX8c!YA67=SH zeUJ-G|A&p%f!sLBL-RUtml^_pWx`a-94$@{rqPOK>;7etkGq_KMg{#Eei6+bkN+_q z!2l*uV8L*k!JmtuW@+7)_1bXWWA>}5ZZxI}zrhQB5a0Kji%()64$wAGBnrM-RcSfU z@uUH60#1r}>eYfm%tZmIg$S#RZ+MdOSt^1F3%-pDc_}yBzk~V9D4aR~f! z8S^4ZC4=}>TxNnvcu9WxuQ5B0b9jIES9Z%&`O}Xg+rDMV4CZiF%5^E3j92^?tca*K z+c)toUyp;vEC+x=1yNg(+=>`cI4QP&Q)QLI_!&fb(20F=!?W$*$Lw?^v$BwBb; zcy+mbJ7y=c`sv4{=?03t`;oIpV1*%^2llc@Huq|+iqUJ@70rL+w>K*iP3nffXvqH0<(E0@{$dZlslZyA}Ai zN_h+nVvYF?UI-i}P;|C$f#1RqhUlMg9~zy_9_#sUL3bLIT03sg1GxbK+mJ5mZKE{r z-f_gi!zJZSh{j38dh1UYSX>}E7g}JqZ#Nud2*sbMc;i5Smn>Rtwtsu~V_RBmG&-x) z%_j$HtT4j;=&Qq0jjz5AZ4F8^;)*hvGugsG$GHWN;Bc}d1aF*5?J z?jEuG!8K1K_U`+H%>*a_XDS6nnB9QRlvO6u*5VLn7Ks2u(p}!!Wdur_WU;vZUrRlB z5;S)qLF-9>-AwT%sMrr|)26UzRU>sI-uDQ!Mxj zOf>)&j!}Z{TWADK%3ro2UdG^q&`58J7`ZfJ;CmRM%^l6lgt$%sA2djY5ISx(@x9{} zF@3HS5<*vYT1-Np3SIieRjLR*yof_^DSTgqfHs(1?LHUwEvST%-bKITF{ ztJi9OHh_hyDb+0EYMrYAE{`}VERu2+CCQYwJRC!m1nvei%Q+$G<{(VKkPbynt!&!J z6%p*IiE(rU2O{=6!3zP=C!l9=h1he{nMAUMqj?VLn_wf485har}Ym&Eeh^wSrc1b_0k~9u~WiB#Ez@1{`a5%=sc4BEJjcl23YW8kE z>#;+|6$}Q?n<>!^ldpjPwvP%;15f4mXY8@wfC<6fqB2!RDa6KQV8FzYn;7eW&^E?b zH;^AksZ?&2z!YxX$Qqx|4Xs;BL>s`666a<> zi(WtpUY3QPW!=Y!5FNI`G%~-BJl!aN2q|OHBzkyx(ofyYEpjjvr0#^v0Cxv%!?E&c9Etc~}kxh>ASnf9wJL9yv3+j|miOGLX@ zPp)lUQn&^dqaV>5Fz{QYG~DRx#>I|YpE~k7(NWi7-=XZ&0_Y!6q=kx0I8dq- z!n|P@09cZN1LS&y;f5~mU1Qa!jj;>yqEBl7xYibN%{^3TGp)_)-JJ7L;=D8a!f23o z-5)9JM#RaB*zN&+nJ`V)1I)*NgAt|T)1W>dO_Oy}PQMtLkj4$@f*^RQbyj96WI}mf zV9LhyTsBWc8mkc!NL*vwl9VuN&5IR`sZ%6I>C3e~0PUHcUKv|sYs=T>Xx3`#ogBC&dp+D6xSh}T=aXa2hu*Jp}TR~n)%%GkA z-?=;Qt%6FXRRde`=0lqVxE9)%D6L`<#tC0!Qk6*r#;!hf-vi$VOs_}tI~&6-`)1JX z2(^UF8Vv@`#M=bDK*N(|Hrb_4EZ-Y_Xmf1x|Do{_jny|D7uHOF*jpm^795YIbuj9# zf~3?Db9;e*WG^B2(bZ3g`|_$pn_n}BT!k%XGBlHHT|u&pNDCG5T#P%pm`Pt85|}`> zt!s+Ug>`qmYzH4t!M1;lySx0jdu&~t-96+zi~6f;o`JgyV_vAf^ON?#yY~o3-v$ze zd(9vqj=0)_5lN_jW;k&X53n3I^`NwK1VcM5WP1#epL#9eLsucafpI-R`}m#&S;6lV zKWxs_ZfyrWoz5oGZnry8x3{Aefj8;w{cYS{vtqo4=Gg~bD%oQ;%&Nw8=p`=D!)oJc z8`|-vF~T3}iTmeGemu;D5pO}Xj+D@b-)q&b3Ao3Vh21=VFkBmUf-C!&P|Q%%4DLq} zo#8vY2-RWnMvf;d=|H$=cH)A(_&{``lqc&Qn0#t!tW7*t_4z^s^&a{Q*)i7tocAVQ z_)KA+>G*WEZ$Y>zk%wq@nBiL_^X{Hn@G~|D<~%AfIl!5lL|oCGP^FSZhL!q&!Z1bu zBUwSieb+XBk;neURh8rY5nVjUc~K^uZhckRDw9{4zN}hT38@6xRDyOPg4bTWeS7Fi zNUy}rPVw%*?|?~4v34XKKXHQ=`Z+-MNV$D*3SQHI8Rx+jFR~Rek&1XBYCl!$!Jc%!KkMStyA0!kzw<_aalYa)b=ai1-{V>)Ur?ouk2l;x z_3*9bbsu}eNoy|lMPGW`V;;u?51pMnt(S}JPZ-RRo7IC)Y)(5l+D_TA6*Vf2{2v7x z4lea|c|zzzyf9kNhFbgm7TzNM|8QG-HNrA@SH5uV6=A(ipSEZN4hAdSIY#f>C+~|z zAr|_7{LqgNu$@EB>z{3tuPmk@-8%^RP*jmNp}X`+-!j7}p6>0F^)@znRNHdRNc>Xz z=+szjO>Gxg=MWyB2M_cbonYNWuM;DSonD;~hbu@B@bD9_6~JOP)wy4iut7h~JI)=G5|(Ra*BZ~c3J>%HSBL!{OvKc&x>hPGr|AGZmfmdioE z{^~uX@zhRwMJ({Fk2F+Ab4n>w)h-=SQ@gOywhi3_1w6UE*>#wJPv1V!8osD*S0Lk? zPpfMg8t#e^ZIzcMoa@0C_e2|?c~fHBQF;ED3hy3M=2ax;MvGhpPQIx3aA6dnd)tM7 zrs^S+&Wyd;_+-!7MmO;$VVQVru03Js_u#XFHM|{UfkteD+O0)L*jj>S}N98aF;FFyi==l>Z`Rw7Z>7_oT z`GXL7b2VHGI`qO3ZN%%^T|kW{1*CjHc~fGi_yj(i8fk;M zE``3JU)x1Xo-Hsmhru-t#?2*oHJCFBWSN}TIA(ZP-qMWzd#YRg%2vOs)r-1+t)FvR zT;KNIt7rW?XT3{e-FSUxRm~GD`>xPYAfSCp9SP`9puQ&Lo>)LjD{|9VW!G)d?^g<- zs{L{5_-V1IFxft#oyB3T)j4~nVdm)3a0o3VwwfqVA3;;gnr2MhiWvxFdD0R?Zgy{JNy=pjEFFOmw4 wIuZZ?cRm0B8UO$Q0000000000q=D*_LLx^3Tp*L6ARCjWA`k{_ApigX0NWM7!vFvP diff --git a/packages/claude/dist/zpwiki-rag.zip b/packages/claude/dist/zpwiki-rag.zip index 16512170022b249b142a0792ee58d705dcc9fdb3..2f66ed639db49abfa947b89375c2bfcc0747d555 100644 GIT binary patch delta 5179 zcmV-B6vXTNMx{rvXAK42c^E1?lWYwhe@kw|FbqWZIfdbM;fngZHz=B24`4_Xl?aU; zN48r8LGE6~Y0=M;GlNHJblFr@?H#_+5V}mKC1ncrQ#I#>9&TQs=$6CW^~XR(uiq)B zm?(5^m_8`iYlwzj8df{I-M2L7^LR|de3M^ap10ek4|gx715HJ~l%e`%mY|59e^_ME zfb$%kQqrKdOmJcvTx+?tEvl9Y0MI4?uz;G=NnwN z8VNL!6B2zP1ZAZ+Dn{A1-WjH)4Gh3RF?D&kREUmA6)Z9a{y+S*P(FXLqqIEofJ#Wg z(0gHpN0Jc{ut1`+ETZwsF|<}WQw}DgWEhn&AhAUef?~lV;N(1cP z+~E4ZHBY#w-gV_yrnH7(m4D_M8rz6^bKQIbP)h>@6aWAK2mk;8Apq2c+Gb@E0037% zlidy-f6JEJ$aVYviVBjGL&_A@(#+T=v}s#bdd7-p)P2+~C!Tf(rwA0uYJw;j1+X8Q z{)9RH1}(Cfg?Cx$9M6WD|H{4f0IGl@)l!sT(GAq&)~)-#w~A2|ovvw?jY!H0mZvOF zZXV7{#z>NJmRCeBX~|MDyCKgqTBnT270)tBe`ubPYRSkq&rZm!yulB~$W*JRpw`pL}lq9UQ&*>iG-RGl)vMh(;t@l(bml`|#lO?CII7 zQ`P-C5-aS@MND&f#Y+63>lt6vqL%of5RC2)4h|GifFa1CBWxWUT_>zS@(w2l1RmhU z8E|I}w4w_}I53P9^(^BFP=^B5NFq`;e5#Qm35ej%HD}GLFT_ih z@{%Rhz$eJ~_h)A(w(s{K9~mnRHYuCKvbrZW!v~UO^AR-vWklx1$l$Ycb3mP^u_j$zDw7%wwbClJu#gCrUhMHE`JBVg9S%H^`L7%jMHWE56^-J_98jkJ-{prc`_zy>a>a0Ku9)p5*p!Xpv zbsdD@o<0BO==jxH2d>fce^@f0LyGnk6n!|nFRovIef;k+8aShuU>i_<2+zTAKgPxi zOiy&@gBu1IFjfqt73&C;F>QKK(_;mum6H)UyD9AJ(+cP_tE-?0eU$h##;idT?X&~A zCs!Z~AXKiy9K(k!GOlere)(Gkee4S$j2g>i$-wX^WktfG^+np^e}5U>^;uP_u{d=p zWQA5MGzR<8+$pODt`O&Vy3!e2fN8%HWqO~n;LFOJU7d6A@@t@esGsIHgJcYLub^O% zfCbQ&;I_jw+i4zlS}3&>@ZMe0JA_6HuV~x{o{5ASzBaT~<5dT?M7=pdALIhl|6!v| zAUEbkY+eWMQbV9jf1C+XpvCFIG+NPaJ-jUPahEgDsGwiNFGX|5(uwb~& z=+9-;u(WB*du=%HG5OV0HyTrg-{1v5i0^yN#V0Wj2WT5qBnrM-)>$>s@uUH60!|h2 zG^<4=o3j#93l`To-B6yUieS{H73Vt0q^hrN^WT;e)^H++qZB&mhW!PV5sKns5I;CZ{WrSEb16OH!14nQh-lmUBj+ znXO5k%-#%u0F- zeUbtqqM$3FB3)JW_D$Ae;w{DU1%UAuLjC|h0pjgOKnM%yHl$PgzJ(CHzTCbYlM_+@ z^doP(fg4ez2uS2y}VU1dd<3``ET_0W~D?^b;Dn@mlhk1&T4t{$$=UxjBx*W zvjpO&0G1->bEc`3QZ!HRzNfOw|63GxrGY`hG*PtAgwsM?D&r!F839&zkJ$a-nx_$Y z_dO>w1`0rl%s>%lH=r|BowLka9OBH9GcY9G<(*w7ptL+s+4cWg>cNwsxr-IFp481G ze_eu#{XjN8g#)V^sZ&C(by*tAi&hKmx1$h_v^H+z42&yn^a(P>lCHp117PMDCFs6| zM!=-{r4R8k1s{Y)dQ+yzr4a$&LloN7(Y&0oYYzCJK{ABUajS{%9j}P#b1jh&y0YUj z34JPb=@(b267=vajlr$NV3+tt`vO&)f9NS6bXbq9(jWuw!xSD&xLlz(aWlI}in>e~ zqJMtj`6y&GK*xL@Ix%YSETwiCexRi!z<=AP3QdDR<@aagvEG0Q!QG;=s*I`-8<&9r6Gv`RtOG*Z7+>8$ew<`Nx>W*G zxOF3Id_F&XJn4kBqt1ZOCT%SJI&z~-nywo+Lot1ewLNtx*H3ktax`vKWqv?66xh!W4>9vnjQJIA zS4*tPayo@Jn{RpCe)aNen|f(6Ghe3YH|z<4P57p6i4sQm0EQ}YZU=bu0!q-TD)lVu zK1K@Bp%12&`F-T+MnOm!e~YG~N02vN^1vjB*ax6OL-K3V*P6+G4r!H(DRJ92&MDCk5-U59;#vac3E{~#R-fbi*dRZeXde;EdgWa@Mf{DQ{y zyd8egjK?z-7kE3|6l@#RhdbvO=cbodSV&AfR9wP=Qe_b44Z8rqk_;Rm z*CPx!baC$*t3E!)F2svIsl(&EE#jJcsNgfrXZ3E*g(z{}nSEh2NW1Qj6m}!x%8S_U zf%-CGnyv?!j|U@_e~M3|=6p2cYhF#i7@3g94d|jMdZ~3*c`%j9Sxj1!I~NiKz7DS|5P+Oi!VNOx5)^QavW214 zSQU2$W%Qqqz*sN7vqhQV;5XAh%kTba2=p+x!QkhfF*k60EZg?;#~01n%s))6{Ol;9 zdfV!jp8Vpjx4Cn(*cioiNG2DdW7?J<-~rXNA%1<)t>}ILh1(N6{Q?}->5X!HZW-7_ zaWFJ`WJC;ce^RoCMSR<)UHPL(B`vuB284>+@m7K@HhOFYc~xQt?dt!XyYt>EsAO6- zuoZ7U_$0u!(7q&Dow7LPbdd{L@dS)rbLzeaeh8RePtEUa47VJbp>9XWC1loUFw{)E zP0$N80$FC8UFyX0z0rp@$0q+D8XwVGebaGa&4j%re{yfZ30PVOqwXq5N*yu#3;ZK_ z3AvB1enQ+=)D_zNhB?Yr*m5R8GfCDpB+H02lL;-@xRZ;S^u-~82~>SulXT9kyX$2; z_yh{J{bStS<;UG)o8s*5A@5le=gzymcynV6hB8Ww9`Vi#}I|7=K&wO3KZm(G}UPJTjgD#cqF&k!8V>4lknWuy`XU zkd<^G+%r3IL0)_yI!Q+PdIu(-S{fS@k5zrTU{SM&{z7()^*`sm$rmA0*k?LEo$Xr? zZb}p&njL2NmWZOeCl7v32GN`*6($EbbHmw{x)UlTU*uS+k5m{Y)&E4)&~V?iP2{nE ze{oqCcz;9}4+>gVoT^)2b-v2QRjx0qHdR7Zf@~{6yAZ)^FTuV&bX7>N*v(Gy?!fPW z;T2mul8&FaK@0sHAP1z}J~#!hX~2y0=!%v(&lfVX0kFQ(9FQG+`at*IHT+mDKyG9# zU9iSaHF~foo$t@O`1CHrxZv-CQCz5af2=xeQrz!xEmJ6{(#9tkZlQXF)(X0hJ>jG^ z7yF_wz3nkg(?NjFPM+4wMfN8Q=E%+J!6!DSog8ha?AVH$DvkUf1sV=6&2@Q&(1&8%Kk-@tEM`-k`z1MHo~pxDa}Qp|bDqOULr%R) zVh?pOw!QOKYKn-VV^;Onzqj5ye~vOl@-F#FeYP~TCEMn>jnS-H4*K=i;2}+*cG4?i zfoFZBp*osV36V*^bfB92!iH}fx(5n)a(lDuFae*weNbzJqPksyOlZDZUCYpLSA=M* zylTR^9(-|6wDFm@CAJ-v=Z~rI?lEOSMRIPmh*jj|i+T?iMgh9FUDQ-Pe`L~`u{Rr^ z>^a*As1z20W<28+SGQK|$&}lAan~t-xc<%^Xc-w$vu3>}q4YN9weDNb8~C2}j516D zKES!j6mxrRpDJ&NugH1+`WTi!3#cOP%wUO#vc zws50X7**-nUN15X!8oyie>nhO+2MhnKQRn?_-pmj976qpv7$YFf`5`dO6Z2sO5!OF zxaD6u=S>yM*c!f?D>LS}O3fSVA7?y+EFKTC<`r~!M`uk*Ose1S+47V%*6K8?gcZl+ z%R19W#MaILhc!5JNtzT6L6nfv0_DvSf{MT3v#kNQnCnvS3-xQle`HDX1*RS_xWU1= zagJA?DN%ts5%UJe9DfU7X{P=Ms(1a$UB7DA%POoNcUoNE{obo}{X2ENOK{zMWM|bd zkPL*b2&zCw`^+ZN(VswlO~^g5j+R#BrtvziTUNi%DS@i?$G79B<)X%7!ink`q)a9y zSHA?E#&If=I37APe=(K{Uo&DBE__O?MrC+6}(ADlVXe;r!h-P-Ug57LwzeP3lErIF-=jx=1)dQC& zArWOjM+%gd(?K*s#!nz}8oJ8$7?CzQxFbsw7=P5?LMkUTqo8TNj;6zyVPb2$@pBx&93zPN1E&9bS&e1m7gB&A0OKxeS5haWLo8qb=dzhf7hVN zQB+ph14Yf`ob}wHGFnsz6i_1uc0m~c=zIcDiGzjYB9h^tw*@P6;#yAgbjydbUecZ} zS9k!3kulnoNp+ZXXG4e-g>)A@Ay^~mMVM3W5Mzqs#5xsC604K>AAVk>UVhlwqRu+8 zGu9}M(V&SeHZub=Q+B8_`{+n9RiYyeKC@Mvo$+9__sxp_+}at+ScYZjPnY@SIBuo1 zpuU?MY52G1iC<)(e*MZ+wlHk+-?@PnTixtFcVAFT0|XQR000O8001EXFOmw4IuZZ? zcRrKb4jq48liSE~@B1qzNGe6j6t&Xb*%jKfEh~Fx#dq2*?aGO7y+I9uA&Di3fiVF0 zL$^QS%D+JmdAJ9^;}f z8ZMR0?7OU63j;FeM7*S(-#Z9S&4uk&J)Eh-G3U3g8YC762pCA#2M55W$^m&YD$U z$d@8c3K5lqHbKU}KRY|IeZL3!NJU|=iNze2)jgpZK2Rc^kD&Q4BR0=R2A{QBjM!4} zSQHAeyMi?g5PS*&VwC|7nE-@-TMNhlkiHe!^sK1F(2_@H^JJmt0+b2xyr7SjxM_br z%498MRZiGEmAnK#e0F$99iSBQDqNOjK4CK{Q|S7Vryvc~`&(LLEy|^g>n#F=co3z^ zL3qPc*etU5u5OzrsmSag(yX~83f;+gipUZdT|y$JAU!y^RJNM!Eg*{Ci!4k z&rRm_^kxy-M{(0`zy}ooRzblZ*lB;mvAE8GJCzcJ1K3#rkkaT)S&&PWTnD7>s50Zz z@cB2#|9Uh)$YX37AdcZ0m^Wnq%ns$@FMiR-gILECC0a5YM8u&m7sWbJ3T*=k2Z%{{ z`18ZVppRWmPE`Cc(pza%K&a!e6xV={cTZq_KnaozP|hMT&~U)$B~dxthwpzMaiB)G z4Oi=nyPrN70^yrAI6kE*BL}_*4G+nZ3{X$vP?N4I$fO2Ct+b>Tc5+W3^c0{SNa&!q zmQNC2z5E))P}2%~2f>U-EASE_=#v)ET0+N#ehD50!_mE`ee4S$j2f$GDZucMvLa#8`XYaA@xKc0`m8Ew zEKXfYRif4Mjlq62cVyMT72=$hu5`xcVA`)_5#MJl__E?=SLYnO{2Hk5>*v|cAR2?+ z%Q+Y%U;(s6xNSepcAAHs7K(NP-n&bBhtO!@iN<~4sf@VcYfW1kPaW72^yYYdkPA%z zhmF>O+&IZY^Ez;s8UlZ1!c@u}Elv-n(TZm4{$-JmyPSbW1^pU+5zQTs|1loH047ji z!El?wpNpVoY2B9f+Hl@u_N%FGG^Pr_!3%y6-}jn}PhuVp&^Ayc3cgxZX*tmGqycUM zPKtQy)q+CIMFFXW2&;^5c#`s2DuM|MzKsicDL31{gZazlZBl=3-xN%72>fyx^CC$l zgZNZjW`aq0Nq+jTF*}ZPcz^d-cFR-w(~lzCzGcY_=5SWZbt##QSNs;Nh^RK(H}NfB zkAuc62Y^8ZQCpGRiWpHiDYk!8WtGGD8AN%|iG6azv+du<>~tlwvXE(Y$?_surrS44 zWQkzVELJd*LtKA-DV7o&h-BM0QBt)Q1jU{uF=SSHHHfQ$;F4tk3Lsy?y6f$mpWd)s zmW7yQ+aH+B0n8@Dg<6m-d&uS>v}2&x?W*cfOyvcSU;{|3Z+Xlw5xG^d{h`b%Ce0L% z`&o+uw+TzX`(Bg`_@Gh|ujHn+EQRYJ2q8piYhFf6TH1ed0m3h!AX|YAp21cBpq$28z7W05)$q{uBEs%gq z!D5MmW8{Bi`&QM&&4gNQ|GQ`{qLVsMnoCH6wCL3|?D9_n+K{-ld%cN&yhJ8sYexd8#&kS^+Nqcrc{am2yH zCFMrWS0Tp&3YT41+tHymUL#h<8n<3N9xELv{1e|z_1TUu;1I;+&pCkJY* zFv9)g%@T+o16Yon&xEE@%+Wl(`<|;V|8G&)NdvuvsiWvN6Haq^NybGnGXkvc9q&pzOz|bC*bi*erm$yKBXuO?T9=ivylA!1eme@`NNepj&cL|xTAv_OEcgmcH2@Zl zQG)JUXar2kU$!A$#^8g{NNA77()V;MO>T49ewD8^cLK8?@Ux;3QHl@ z1_nBWnV2fjh=DfcGRaD+{EXO4#y3;M&BQJK+{{31(i>aqdTDA{HHKzB=0ZQK*J^(@ zfQ70l)hyy_ovQ&Zk2onTl5!O#$&|J{97B`@?glf9Id{AF z@cPaXC6hF3lDBe*tE60ZNk6NSG!B1dE;2~Konqv0IL5|yVreFgY?*Ir_HI7wu|vic z3GF3(?#KvV{z{HW880&!0HpW*skRL~> zRBn~P6mH$f8lTS(A5S`A?Wi;0vxysqc6&dcG%m@CZV3DW6H{eEB=3gcq|J@$&1>x*-l#bkxNz--XdMKuEp|+_$QsxJALxKJ5@DMXE zV$82_yINvRmeVP;*?gJ-6($uDPz$jdU$!$B@ax3hlXX%~zZjX2#trC#Ab6>DR%R(=LU~?b%Et6u zHcvzvs}T}NTw~mllrU<|ixrHiQzS;|%e6iL?U|ll8Czp(%h%@Qw>1Km!!0xw?&hf1 zEGjx>ZmSuz)tY5Vr>5-I5NRa7JUR4@aTrUK))Lxyg?kqXweNp#X3{rUV*1}ZxC8}U zqikU)HKyXupa}l+5g6;mceW@q9QO5Cet-w0X+!+_qFd4Z017uJc=`o6s?!^C zd~O-oL~$@Qcw~P>3~?ga!y>+I)2{qcKuHVkzXm~ZJKjpL#l{v}L0(DBpq>8TxjXNz zf=Z@U16%RtLz@J+7TT96tzr?z314JVl}QA~u0D0&1K$TsuSfGc8^bO8X3*^jwS>$X z4F=7`+XTHp!;@t;*`-b_-y3~sb8Pbeq45!o)i)g%)=YocTO#)s9FL`SFzT*?q|^~} zdx3vsFCq8Q)lZ1~@~T9eUo(eXg)L_?G?Q#yL9&cU3l;HPj61oQNnacim_W6yYl_c> zb$7jN2Om$twttMfyZpF&Y+an)J>)%$`m1Z6fx8Q1UZ}qFllH*7_XtPd1`>sP%^)C- zxY~jdNvMBjIB^jVupBn^ptN%YLpv>Gdkm4EdM)5XS0TNDaXmo$_?`q=!S575Y|hkf zZ3jJ_&L+}sw>wd{x1$w-H|gyCZQNe7V!Vdt*#})J*<&`$s>XEaB`(mzYU61e+VQ3_ z!XN61`{zx5Jj{g=Z$Y$(vLsumzy`qjO0!3H@aY5Hd)M$| zwE(%1ig+PvKUM3&o^-xH>*CY94C8{o^G1JhzTz=;*rd4M<60(PP^FEJH{3$?@U7)_ zAA7<{YcBRhUwYeP9>)U@ot-?bmy7IA7|fBI)q_uLPCGf;PT8>)H7bq#9|al?F7%R#^X>OG|K z)J}RuEby$4G*m}(N-0y-E*(%)yRgx=4c!9;Jh{Eub(nxp-#*Y9zNl_jAmf`)t7{n= z?urm?m6s-*>%kZIL>r%ZQ)1gudH$FR?;ca;RV3#|i(CaxzNq(bVHBWy+l7Co>LHWP zjJ?_TWY5_~H}NK6nRsljJz?ng;Io1?yd7kLMr?!RZZpjoA@3|SOuFu`_vaqfb`Rou z0QxQO=lb2ptN-f<5BwHj)G&)mv-bLpu~8UDXMP7 z-?!QFjWuR;+Ec)aWAm@^7wnVi=+W_VZL(v1Fls$2caR==v%i@JZUpL1GV-}c_C zXZ<^8y-Q-A3K`c~MB=k$kX&A;b3d5l@6J!36n<=&yH)0tyscG;VV?mn$H&Jx_+Gw8_ z%)^J4NgzBjfe>+Ov>ktcw>#&s;c1|MQMv~PMhOmtbKl|Pg#*wQvG&2#r$8g}UQ;>% zvD3fUJI{Fd5#tp8O+j>JAwv2Qm7a(c13sAf^baAZ`f8pD3mYfgiAh?a0nQxjj#l>7 zIeVsI=IGII2rVSGnkY~oK~u|`ZXW3~Xll@Bv;@EAGa0F7ak5+y^=yBIx8B-*i)!dw z0M!T1)j|`?dM+12q9%Zj94M`(gJ6V=pFm_Wbd~EdB5ic2RV#9ii8SR7L6cx?7~=1B z!jSsI5cMhy&2DsTaeAXb8IP`$azI)K3;qwYTOoh}1zalS7Cn>TB03x|k_wGF5&!^q rJ^%m}00000000000001_fyEA!-wqd(O(QA+GLv{CCkALD000009%H$y diff --git a/test/test_answer_health.py b/test/test_answer_health.py new file mode 100644 index 0000000..2343272 --- /dev/null +++ b/test/test_answer_health.py @@ -0,0 +1,54 @@ +import json + +import pytest + +from tools.answers import model_client as model +from tools.runtime import answer_health as health +from tools.knowledge.publication import checkpoint_state, PENDING_FILE +from tools.sync.index_state import durable_json, VERSION_FILE +from test_index_sync import snapshot + + +def test_generation_poll_never_calls_provider_and_reports_unknown(monkeypatch): + monkeypatch.setenv("ANSWER_API_KEY", "private-secret") + monkeypatch.setenv("ANSWER_MODEL", "test") + monkeypatch.setenv("ANSWER_LLM_URL", "https://provider.example/v1/chat/completions") + monkeypatch.setattr(model, "_status", {}) + def forbidden(*args, **kwargs): + pytest.fail("Health polling must not call LLM") + monkeypatch.setattr(model, "_complete_request", forbidden) + assert model.generation_status()["status"] == "unknown" + assert "private-secret" not in json.dumps(model.generation_status()) + + +def test_probe_records_success_failure_and_configuration_change(monkeypatch): + monkeypatch.setenv("ANSWER_API_KEY", "test-secret") + monkeypatch.setenv("ANSWER_MODEL", "test") + monkeypatch.setenv("ANSWER_LLM_URL", "https://provider.example/v1/chat/completions") + monkeypatch.setattr(model, "_status", {}) + monkeypatch.setattr(model, "_complete_request", lambda *a, **kw: {"choices": [{"message": {"content": "OK"}}]}) + assert model.probe_generation()["ready"] is True + def fail(*args, **kwargs): + raise ConnectionError("do not expose private-secret") + monkeypatch.setattr(model, "_complete_request", fail) + result = model.probe_generation() + assert not result["ready"] and result["reason"] == "model_probe_failed" + assert "private-secret" not in json.dumps(result) + monkeypatch.setenv("ANSWER_MODEL", "new-model") + assert model.generation_status()["status"] == "unknown" + + +def test_content_health_distinguishes_current_stale_corrupt_and_pending(tmp_path): + version = {"version": "old"} + snapshot(tmp_path, version) + index = {"version": "content", "index_version": version, "graph_ready": False, "units": []} + checkpoint_state(tmp_path, index) + db = tmp_path / "zp_index.sqlite" + assert health.content_readiness(db, version, graph_available=False)["ready"] + path = tmp_path / "knowledge/index.json" + path.write_text(path.read_text() + " ") + assert health.content_readiness(db, version, graph_available=False)["reason"] == "content_checkpoint_invalid" + checkpoint_state(tmp_path, {**index, "index_version": {"version": "new"}}) + assert health.content_readiness(db, version, graph_available=False)["reason"] == "content_version_mismatch" + durable_json(tmp_path / PENDING_FILE, {}) + assert health.content_readiness(db, version, graph_available=False)["reason"] == "content_publication_incomplete" diff --git a/test/test_content_publication.py b/test/test_content_publication.py new file mode 100644 index 0000000..8f3e004 --- /dev/null +++ b/test/test_content_publication.py @@ -0,0 +1,145 @@ +"""Content is published with the source generation, including failure recovery.""" +import json +from pathlib import Path + +import pytest + +from tools.knowledge import publication as content +from tools.knowledge import store +from tools.sync import index_state as state +from tools.sync import sync_indexes as sync +from test_index_sync import Session, snapshot + + +def checkpoint(version, label="content"): + return {"version": label, "index_version": version, "graph_ready": True, + "units": [], "profiles": 1} + + +class ContentSession(Session): + def __init__(self, version, content_state): + super().__init__(version) + self.content = content_state + + def begin_transaction(self): + session = self + class Tx: + def __enter__(self): + self.version, self.content = session.version, session.content + return self + def __exit__(self, *args): + pass + def commit(self): + if session.fail_commit == "unknown": + session.unavailable = True + raise ConnectionError("unknown commit") + session.version, session.content = self.version, self.content + if session.fail_commit == "response_lost": + raise ConnectionError("lost response") + return Tx() + + +def graph_state(value): + return {"version": value["version"], "checkpoint_sha256": value["checkpoint_sha256"], + "index_version_json": json.dumps(value["index_version"], sort_keys=True)} + + +@pytest.fixture +def publication(tmp_path, monkeypatch): + old, new = {"version": "old"}, {"version": "new"} + data, stage = tmp_path / "data", tmp_path / "stage" + snapshot(data, old) + snapshot(stage, new) + old_state = content.checkpoint_state(data, checkpoint(old, "old-content")) + content.checkpoint_state(stage, checkpoint(new, "new-content")) + session = ContentSession(old, graph_state(old_state)) + monkeypatch.setattr(sync, "write_graph_transaction", lambda tx, payload, version: setattr(tx, "version", version)) + monkeypatch.setattr(sync, "graph_counts", lambda tx: ([("Person", 1)], [])) + def write_content(tx, index, *, checkpoint_sha256=None): + tx.content = {"version": index["version"], "checkpoint_sha256": checkpoint_sha256, + "index_version_json": json.dumps(index["index_version"], sort_keys=True)} + monkeypatch.setattr(store, "write_graph_transaction", write_content) + def read_content(session): + if session.unavailable: + raise ConnectionError("database offline") + return session.content + monkeypatch.setattr(store, "graph_state", read_content) + return data, stage, old, new, session + + +@pytest.mark.parametrize("commit", [None, "response_lost"]) +def test_publish_all_sources_and_content_in_same_transaction(publication, commit): + data, stage, old, new, session = publication + session.fail_commit = commit + sync.publish_snapshot(data, stage, new, {}, session, knowledge=checkpoint(new, "new-content")) + assert state.read_version(data) == state.sqlite_version(data / "zp_index.sqlite") == session.version == new + assert store.load_index(data, new)["version"] == "new-content" + assert content.state_matches_graph(content.read_state(data), session.content) + assert not (data / state.PENDING_FILE).exists() + + +def test_failed_content_file_switch_restores_entire_generation(publication, monkeypatch): + data, stage, old, new, session = publication + files = (*sync.FILES, *content.FILES) + before = {f: (data / f).read_bytes() for f in files} + replace = sync.replace_file + def fail(source, target): + if target == data / "knowledge/state.json": + raise OSError("disk failed") + replace(source, target) + monkeypatch.setattr(sync, "replace_file", fail) + with pytest.raises(OSError): + sync.publish_snapshot(data, stage, new, {}, session, knowledge=checkpoint(new, "new-content")) + assert before == {f: (data / f).read_bytes() for f in files} + assert session.version == old + assert content.state_matches_graph(content.read_state(data), session.content) + + +def test_unknown_commit_blocks_metadata_and_content(publication): + data, stage, old, new, session = publication + session.fail_commit = "unknown" + with pytest.raises(ConnectionError): + sync.publish_snapshot(data, stage, new, {}, session, knowledge=checkpoint(new, "new-content")) + with pytest.raises(state.IndexUnavailableError): + with state.index_snapshot(data): + pass + assert (stage / "previous/knowledge/index.json").exists() + + +@pytest.mark.parametrize("commit", [None, "response_lost", "unknown"]) +def test_standalone_content_publication_failure_barrier(publication, commit): + data, stage, old, new, session = publication + session.fail_commit = commit + if commit == "unknown": + with pytest.raises(ConnectionError): + content.publish_checkpoint(data, stage, checkpoint(new, "new-content"), session) + with pytest.raises(state.IndexUnavailableError): + store.load_index(data, new) + assert (data / content.PENDING_FILE).exists() + else: + content.publish_checkpoint(data, stage, checkpoint(new, "new-content"), session) + assert content.state_matches_graph(content.read_state(data), session.content) + assert store.load_index(data, new)["version"] == "new-content" + + +def test_content_staging_failure_leaves_old_primary_and_content(publication, monkeypatch): + from tools.knowledge import build + data, stage, old, new, session = publication + before = {f: (data / f).read_bytes() for f in (*sync.FILES, *content.FILES)} + monkeypatch.setattr(sync, "prepare_snapshot", lambda *args: (new, {}, {})) + monkeypatch.setattr(build, "configured_extractor", lambda: (None, {})) + def fail(*args, **kwargs): + raise RuntimeError("LLM unavailable") + monkeypatch.setattr(build, "prepare_index", fail) + with pytest.raises(RuntimeError, match="LLM unavailable"): + sync.synchronize_indexes(Path("unused-wiki"), data) + assert before == {f: (data / f).read_bytes() for f in before} + assert not (data / state.PENDING_FILE).exists() + + +def test_corrupt_content_fails_integrity_check(publication): + data, *_ = publication + path = data / "knowledge/index.json" + path.write_text(path.read_text() + " ") + with pytest.raises(RuntimeError, match="invalid"): + store.load_index(data, {"version": "old"}) diff --git a/test/test_git_runner.py b/test/test_git_runner.py index 2c8a07a..7aefcd1 100644 --- a/test/test_git_runner.py +++ b/test/test_git_runner.py @@ -305,3 +305,16 @@ def test_runner_credentials_and_state_are_ignored_by_git(private_path): text=True, ) assert result.returncode == 0 + + +def test_existing_key_file_updates_only_fork(repositories, tmp_path): + source, fork, _ = repositories + upstream_head = commit(source, "pending-with-local-key") + key = tmp_path / "private-key" + key.write_text("not-an-actual-key-local-transport-only") + key.chmod(0o600) + result = sync(repositories, "--key-file", str(key), token="") + assert result.returncode == 0, result.stderr + assert git(fork, "rev-parse", "master") == upstream_head + assert git(source, "rev-parse", "HEAD") == upstream_head + assert key.read_text() == "not-an-actual-key-local-transport-only" diff --git a/test/test_knowledge.py b/test/test_knowledge.py index 8ff5af9..e9ae820 100644 --- a/test/test_knowledge.py +++ b/test/test_knowledge.py @@ -529,32 +529,18 @@ def test_build_is_cached_and_source_changes_cannot_activate_it(tmp_path): assert activated == (data / "knowledge/index.json").read_bytes() -@pytest.mark.parametrize("enabled", [False, True]) -def test_index_sync_refreshes_content_only_after_it_has_been_enabled( - tmp_path, monkeypatch, enabled -): +def test_rebuild_delegates_content_to_coordinated_publication(tmp_path, monkeypatch): import tools.rag.rebuild_index as rebuild import tools.sync.sync_indexes as sync from tools.knowledge import build as knowledge - target = tmp_path / "data" - target.mkdir() - if enabled: - (target / "knowledge").mkdir() - (target / "knowledge/index.json").write_text("{}") - calls = [] - phases = [] - monkeypatch.setattr(sync, "synchronize_indexes", lambda *a, **kw: {"counts": {"documents": 1}}) - monkeypatch.setattr( - knowledge, "refresh_existing", lambda *a: calls.append(a) or {"cached_batches": 1} - ) - result = rebuild.rebuild_index( - wiki_root=tmp_path / "wiki", data_dir=target, progress=phases.append - ) - assert bool(calls) == enabled - assert ("knowledge" in result) == enabled - if enabled: - assert phases[-1] == "extracting_knowledge" + expected = {"counts": {"documents": 1}, "knowledge": {"cached_batches": 1}} + monkeypatch.setattr(sync, "synchronize_indexes", lambda *a, **kw: expected) + def forbidden(*args): + pytest.fail("Content must not be refreshed after publishing primary indexes") + monkeypatch.setattr(knowledge, "refresh_existing", forbidden) + result = rebuild.rebuild_index(wiki_root=tmp_path / "wiki", data_dir=tmp_path / "data") + assert result["knowledge"] == expected["knowledge"] def test_explicit_work_year_is_not_misused_as_an_event_date(): diff --git a/test/test_publication_health.py b/test/test_publication_health.py index 75b985c..f85bf9b 100644 --- a/test/test_publication_health.py +++ b/test/test_publication_health.py @@ -221,3 +221,16 @@ def test_connection_readiness_is_distinct_from_graph_readiness(monkeypatch, actu expected = {"version": "expected"} if reason == "version_mismatch" else actual neo, graph = health.neo4j_readiness(expected) assert neo["ready"] is True and graph["ready"] is False and graph["reason"] == reason + + +def test_model_probe_requires_authentication_and_returns_sanitized_status(corpus, monkeypatch): + import tools.answers.model_client as model + seen = [] + monkeypatch.setattr(model, "probe_generation", lambda: seen.append(1) or + {"ready": True, "status": "ready", "reason": None}) + with TestClient(main.app) as client: + assert client.post("/health/generation").status_code == 401 + assert not seen + response = client.post("/health/generation", headers={"X-API-Key": "a" * 64}) + assert response.status_code == 200 and response.json()["ready"] is True + assert len(seen) == 1 diff --git a/test/test_sync_jobs.py b/test/test_sync_jobs.py index 5c9fb19..45cea73 100644 --- a/test/test_sync_jobs.py +++ b/test/test_sync_jobs.py @@ -176,3 +176,20 @@ def test_other_branches_and_deletions_do_not_queue(api, extra, reason): def test_malformed_push_is_rejected(api, extra): assert post_push(api, **extra).status_code == 400 assert main.app.state.sync_jobs.overview()["active"] == [] + + +def test_daily_schedule_is_durable_and_duplicate_workers_do_not_enqueue_twice(tmp_path, monkeypatch): + monkeypatch.setattr(time, "time", lambda: 1000) + jobs = SyncJobs(tmp_path, periodic_interval_seconds=86400) + assert jobs.schedule_if_due() + restarted = SyncJobs(tmp_path, periodic_interval_seconds=86400) + assert not restarted.schedule_if_due() + assert len(jobs.overview()["active"]) == 1 + assert jobs.overview()["schedule"]["next_due"] == 87400 + # Manual webhook gets the same queued job, without losing its Git update. + assert jobs.enqueue(pull_git=False)["coalesced"] + assert jobs.overview()["active"][0]["pull_git"] + monkeypatch.setattr(time, "time", lambda: 87401) + assert restarted.schedule_if_due() + assert len(jobs.overview()["active"]) == 1 + assert not SyncJobs(tmp_path).schedule_if_due() diff --git a/test/test_wiki_git.py b/test/test_wiki_git.py index 6ddbf99..7d9100d 100644 --- a/test/test_wiki_git.py +++ b/test/test_wiki_git.py @@ -93,3 +93,20 @@ def test_real_git_fetch_fast_forward_and_local_change_protection(tmp_path, monke wiki_git.update_wiki(checkout) assert git(checkout, "rev-parse", "HEAD") == local_head assert (checkout / "local.md").read_text() == "local commit" + + +def test_upstream_sync_rejects_wrong_destination_before_any_write(tmp_path, monkeypatch): + root = tmp_path / "wiki" + (root / ".git").mkdir(parents=True) + monkeypatch.setenv("SYNC_UPSTREAM_FORK", "1") + seen = [] + def run(args, **kwargs): + seen.append(args) + output = {"remote": "git@git.kemt.fei.tuke.sk:KEMT/zpwiki.git", + "branch": "master", "status": ""}.get(args[1], "") + return subprocess.CompletedProcess(args, 0, stdout=output, stderr="") + monkeypatch.setattr(wiki_git.subprocess, "run", run) + with pytest.raises(RuntimeError, match="restricted"): + wiki_git.update_wiki(root) + assert len(seen) == 3 + assert all(args[0] == "git" for args in seen) diff --git a/tools/answers/model_client.py b/tools/answers/model_client.py index cc39a00..9274265 100644 --- a/tools/answers/model_client.py +++ b/tools/answers/model_client.py @@ -6,6 +6,10 @@ import json import os import urllib.parse import urllib.request +import threading +import time +import hashlib +from datetime import datetime, timezone class NoRedirect(urllib.request.HTTPRedirectHandler): @@ -13,14 +17,77 @@ class NoRedirect(urllib.request.HTTPRedirectHandler): return None -def complete_with_configured_model(messages: list[dict], *, max_tokens: int | None = None) -> dict: - if max_tokens is not None and (type(max_tokens) is not int or not 1 <= max_tokens <= 16384): - raise ValueError("Invalid model output budget") +def model_configuration() -> dict: provider = os.getenv("ANSWER_PROVIDER", "openai_compatible").strip() model = os.getenv("ANSWER_MODEL", "").strip() if provider == "anthropic": key = (os.getenv("ANSWER_API_KEY") or os.getenv("ANTHROPIC_API_KEY", "")).strip() endpoint = os.getenv("ANSWER_LLM_URL", "https://api.anthropic.com/v1/messages").strip() + elif provider == "openai_compatible": + legacy = os.getenv("OPENWEBUI_API_KEY", "").strip() + key = os.getenv("ANSWER_API_KEY", "").strip() or legacy + endpoint = os.getenv("ANSWER_LLM_URL", "https://ui.tukekemt.xyz/api/chat/completions" if legacy else "").strip() + model = model or ("model120-fast" if legacy else "") + else: + raise RuntimeError("Unsupported answer provider") + parsed = urllib.parse.urlsplit(endpoint) + if (not key or not model or any(c in key for c in "\r\n") or parsed.scheme != "https" + or not parsed.hostname or parsed.username is not None or parsed.password is not None + or parsed.fragment or parsed.query or any(c.isspace() for c in endpoint)): + raise RuntimeError("Answer model is not configured") + return {'provider':provider, 'model':model, 'endpoint':endpoint, 'key':key} + + +_status_lock = threading.Lock() +_status = {} +HEALTH_MAX_AGE_SECONDS = 900 + + +def configuration_key(config: dict) -> str: + return hashlib.sha256(json.dumps(config,sort_keys=True).encode()).hexdigest() + + +def generation_status() -> dict: + try: + config = model_configuration() + except (RuntimeError,ValueError): + return {'ready':False,'status':'unavailable','reason':'model_not_configured','configured':False} + public={'configured':True,'provider':config['provider'],'model':config['model']} + with _status_lock: + status = dict(_status) + if status.get('configuration') != configuration_key(config) or time.time()-status.get('checked_at',0)>HEALTH_MAX_AGE_SECONDS: + return {**public,'ready':False,'status':'unknown','reason':'model_availability_not_verified'} + return {**public,'ready':status['success'],'status':'ready' if status['success'] else 'unavailable', + 'reason':None if status['success'] else 'model_probe_failed', + 'checked_at_utc':datetime.fromtimestamp(status['checked_at'],timezone.utc).isoformat()} + + +def complete_with_configured_model(messages: list[dict], *, max_tokens: int | None = None) -> dict: + config = model_configuration() + try: + result = _complete_request(messages,config,max_tokens=max_tokens) + except Exception: + with _status_lock: + _status.update(configuration=configuration_key(config), checked_at=time.time(),success=False) + raise + with _status_lock: + _status.update(configuration=configuration_key(config),checked_at=time.time(),success=True) + return result + + +def probe_generation() -> dict: + try: + complete_with_configured_model([{'role':'user','content':'Reply only with OK.'}],max_tokens=32) + except Exception: + pass # The caller gets only sanitized configuration/availability, never raw exceptions. + return generation_status() + + +def _complete_request(messages: list[dict], config: dict, *, max_tokens: int | None = None) -> dict: + if max_tokens is not None and (type(max_tokens) is not int or not 1 <= max_tokens <= 16384): + raise ValueError("Invalid model output budget") + provider,model,key,endpoint = (config[k] for k in ('provider','model','key','endpoint')) + if provider == "anthropic": system = "\n\n".join(row["content"] for row in messages if row["role"] == "system") body = { "model": model, @@ -32,33 +99,12 @@ def complete_with_configured_model(messages: list[dict], *, max_tokens: int | No body["system"] = system headers = {"x-api-key": key, "anthropic-version": "2023-06-01"} elif provider == "openai_compatible": - # Compatibility for historical experiments; new installations use ANSWER_API_KEY. - legacy = os.getenv("OPENWEBUI_API_KEY", "").strip() - key = os.getenv("ANSWER_API_KEY", "").strip() or legacy - endpoint = os.getenv( - "ANSWER_LLM_URL", "https://ui.tukekemt.xyz/api/chat/completions" if legacy else "" - ).strip() - model = model or ("model120-fast" if legacy else "") body = {"model": model, "messages": messages, "temperature": 0, "stream": False} if max_tokens is not None: body["max_tokens"] = max_tokens headers = {"Authorization": "Bearer " + key} else: raise RuntimeError("Unsupported answer provider") - parsed = urllib.parse.urlsplit(endpoint) - if ( - not key - or not model - or any(c in key for c in "\r\n") - or parsed.scheme != "https" - or not parsed.hostname - or parsed.username is not None - or parsed.password is not None - or parsed.fragment - or parsed.query - or any(c.isspace() for c in endpoint) - ): - raise RuntimeError("Answer model is not configured") request = urllib.request.Request( endpoint, data=json.dumps(body, ensure_ascii=False).encode(), @@ -78,4 +124,9 @@ def complete_with_configured_model(messages: list[dict], *, max_tokens: int | No if not text: raise RuntimeError("Missing answer model output") return {"choices": [{"message": {"content": text}}]} + if not isinstance(value,dict) or not value.get('choices'): + raise RuntimeError('Invalid answer model response') + choice=value['choices'][0] + if choice.get('finish_reason') not in (None,'stop') or not choice.get('message',{}).get('content'): + raise RuntimeError('Incomplete answer model output') return value diff --git a/tools/claude/build_claude_packages.py b/tools/claude/build_claude_packages.py index 58bb459..319debc 100644 --- a/tools/claude/build_claude_packages.py +++ b/tools/claude/build_claude_packages.py @@ -12,7 +12,7 @@ import zipfile ROOT = Path(__file__).resolve().parents[2] BASE = ROOT / "packages/claude" ENGINES = ("rag", "graphrag") -PACKAGE_VERSION = "1.8.2" +PACKAGE_VERSION = "1.8.3" def json_bytes(value): diff --git a/tools/git_runner/sync_fork.sh b/tools/git_runner/sync_fork.sh index 2309c6a..3d08fb0 100644 --- a/tools/git_runner/sync_fork.sh +++ b/tools/git_runner/sync_fork.sh @@ -8,26 +8,34 @@ fork_url='git@git.kemt.fei.tuke.sk:jp170na/zpwiki.git' upstream_url='git@git.kemt.fei.tuke.sk:KEMT/zpwiki.git' script_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" check_only=false +local_key='' case "${1:-}" in --check) check_only=true ;; + --key-file) + [[ $# -eq 2 && -f "$2" ]] || { echo "Chyba platny lokalny SSH kluc." >&2; exit 64; } + local_key="$2" + ;; '') ;; -h|--help) - echo "Pouzitie: bash tools/git_runner/sync_fork.sh [--check]" + echo "Pouzitie: bash tools/git_runner/sync_fork.sh [--check | --key-file /cesta/ku/klucu]" echo "--check iba porovna vetvy bez zapisu; pouzije dostupny lokalny SSH pristup." exit 0 ;; *) echo "Neznamy parameter." >&2; exit 64 ;; esac -if [[ $# -gt 1 ]]; then +if [[ $# -gt 1 && -z "$local_key" ]]; then echo "Prilis vela parametrov." >&2 exit 64 fi -if ! "$check_only"; then +if ! "$check_only" && [[ -z "$local_key" ]]; then : "${ZPWIKI_SYNC_SSH_KEY:?Chyba Actions secret ZPWIKI_SYNC_SSH_KEY}" fi export GIT_TERMINAL_PROMPT=0 LC_ALL=C +# Ignore URL rewrites and hooks in machine-wide/user Git configuration. +# The only write transport below is the explicit, fixed fork URL. +export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null export ZPWIKI_KNOWN_HOSTS_FILE="$script_root/ssh_known_hosts" if [[ ! -s "$ZPWIKI_KNOWN_HOSTS_FILE" ]]; then @@ -38,7 +46,9 @@ fi work="$(mktemp -d /tmp/zpwiki-fork-sync.XXXXXXXX)" trap 'rm -rf "$work"' EXIT -if [[ -n "${ZPWIKI_SYNC_SSH_KEY:-}" ]]; then +if [[ -n "$local_key" ]]; then + export ZPWIKI_SSH_KEY_FILE="$local_key" +elif [[ -n "${ZPWIKI_SYNC_SSH_KEY:-}" ]]; then key_text="${ZPWIKI_SYNC_SSH_KEY//$'\r'/}" printf '%s\n' "${key_text%$'\n'}" > "$work/id_ed25519" unset key_text @@ -91,5 +101,5 @@ if "$check_only"; then fi # Normal push also rejects a concurrent divergent update on the server. -git_cmd push origin "$upstream:refs/heads/master" +git_cmd push "$fork_url" "$upstream:refs/heads/master" echo "Fork bol aktualizovany na $upstream." diff --git a/tools/knowledge/build.py b/tools/knowledge/build.py index 18fab1f..4b66e3b 100644 --- a/tools/knowledge/build.py +++ b/tools/knowledge/build.py @@ -55,9 +55,43 @@ def build( def _build(wiki_root, data_dir, complete, model_identity, *, graph, vectors, workers, progress): + import tempfile + import shutil + from tools.sync.index_state import index_snapshot + from tools.knowledge.publication import checkpoint_state, publish_checkpoint, PENDING_FILE + from tools.knowledge.store import create_constraints + from tools.graphrag.graph_db import create_neo4j_driver, get_neo4j_settings + from tools.sync.sync_indexes import source_files + with index_snapshot(data_dir) as version: + if version is None: + raise RuntimeError("Build the shared source indexes before content extraction.") + index, summary = prepare_index(wiki_root, data_dir, complete, model_identity, + graph=graph, vectors=vectors, workers=workers, progress=progress, source_version=version) + stage = Path(tempfile.mkdtemp(prefix=".knowledge-stage-", dir=data_dir)) + try: + checkpoint_state(stage, index) + with publication_lock(data_dir, exclusive=True): + source_digest=hashlib.sha256(json.dumps(source_files(wiki_root),sort_keys=True).encode()).hexdigest() + if read_version(data_dir) != version or source_digest != version['source_sha256']: + raise RuntimeError("Source changed during extraction; checkpoint not activated.") + if graph: + settings = get_neo4j_settings() + with create_neo4j_driver(settings) as driver, driver.session(database=settings.database) as session: + driver.verify_connectivity() + create_constraints(session) + publish_checkpoint(data_dir, stage, index, session) + else: + publish_checkpoint(data_dir, stage, index) + return summary + finally: + if not (data_dir / PENDING_FILE).exists(): + shutil.rmtree(stage) + + +def prepare_index(wiki_root, data_dir, complete, model_identity, *, graph=True, vectors=True, workers=2, progress=None, source_version=None): from tools.sync.sync_indexes import source_files - version = read_version(data_dir) + version = source_version or read_version(data_dir) if not version: raise RuntimeError("Build the shared source indexes before content extraction.") sources = source_files(wiki_root) @@ -210,15 +244,10 @@ def _build(wiki_root, data_dir, complete, model_identity, *, graph, vectors, wor if vectors: index.update(embedding_index(extracted, data_dir)) index["version"] = identifier(json.dumps(index, ensure_ascii=False, sort_keys=True)) - with publication_lock(data_dir, exclusive=True): - if read_version(data_dir) != version or source_files(wiki_root) != sources: - raise RuntimeError("Source changed during extraction; checkpoint not activated.") - if graph: - from tools.knowledge.store import write_graph + if source_files(wiki_root) != sources: + raise RuntimeError("Source changed during extraction; checkpoint not activated.") + return index, { - write_graph(index) - durable_json(data_dir / "knowledge/index.json", index) - return { "profiles": len(documents), "evidence_units": len(extracted), "llm_batches": calls, diff --git a/tools/knowledge/publication.py b/tools/knowledge/publication.py new file mode 100644 index 0000000..25e9096 --- /dev/null +++ b/tools/knowledge/publication.py @@ -0,0 +1,119 @@ +"""Recoverable publication of a content checkpoint and its Neo4j projection.""" +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import shutil + +from tools.sync.index_state import durable_json, fsync_directory, IndexUnavailableError + +FILES = ("knowledge/index.json", "knowledge/state.json") +PENDING_FILE = "knowledge/.publication-pending.json" + + +def checkpoint_state(stage: Path, index: dict) -> dict: + directory = stage / "knowledge" + directory.mkdir(parents=True, exist_ok=True) + durable_json(directory / "index.json", index) + content = (directory / "index.json").read_bytes() + state = { + "schema": 1, + "version": index["version"], + "index_version": index["index_version"], + "model": index.get("model", {}), + "graph_ready": index.get("graph_ready", False), + "embedding_fingerprint": index.get("embedding_fingerprint"), + "checkpoint_sha256": hashlib.sha256(content).hexdigest(), + "checkpoint_bytes": len(content), + "profiles": index.get("profiles"), + "units": len(index["units"]), + } + durable_json(directory / "state.json", state) + return state + + +def read_state(data_dir: Path) -> dict | None: + from tools.sync.index_state import PENDING_FILE as SOURCE_PENDING + if (data_dir / PENDING_FILE).exists() or (data_dir / SOURCE_PENDING).exists(): + raise IndexUnavailableError("Publikovanie obsahovej extrakcie nebolo dokončené.") + path = data_dir / "knowledge/state.json" + return json.loads(path.read_text()) if path.exists() else None + + +def state_matches_graph(state: dict, actual: dict | None) -> bool: + return bool(actual and actual.get("version") == state["version"] + and actual.get("checkpoint_sha256") == state["checkpoint_sha256"] + and actual.get("index_version_json") == json.dumps(state["index_version"], sort_keys=True)) + + +def replace_file(source: Path, target: Path) -> None: + os.replace(source, target) + + +def publish_checkpoint(data_dir: Path, stage: Path, index: dict, session=None) -> None: + """Caller holds the exclusive publication lock and content build lock. + + A failed/unknown commit leaves a durable marker; readers refuse that + checkpoint until a successful rebuild. A known rollback restores both files. + """ + from tools.knowledge.store import graph_state, write_graph_transaction + + state = json.loads((stage / "knowledge/state.json").read_text()) + old_graph = graph_state(session) if session is not None else None + backup = stage / "previous/knowledge" + backup.mkdir(parents=True) + originals = [] + for name in FILES: + old = data_dir / name + if old.exists(): + target = stage / "previous" / name + shutil.copy2(old, target) + with target.open("rb") as stream: + os.fsync(stream.fileno()) + originals.append(name) + fsync_directory(backup) + had_pending = (data_dir / PENDING_FILE).exists() + durable_json(data_dir / PENDING_FILE, + {"staging_directory": stage.name, "target_version": index["version"]}) + commit_attempted = False + try: + if session is None: + for name in FILES: + replace_file(stage / name, data_dir / name) + fsync_directory(data_dir / "knowledge") + else: + with session.begin_transaction() as tx: + write_graph_transaction(tx, index, checkpoint_sha256=state["checkpoint_sha256"]) + for name in FILES: + replace_file(stage / name, data_dir / name) + fsync_directory(data_dir / "knowledge") + commit_attempted = True + tx.commit() + except BaseException: + if commit_attempted: + # Never equate a lost acknowledgement with a rolled-back commit. + actual = graph_state(session) + if state_matches_graph(state, actual): + if json.loads((data_dir / "knowledge/state.json").read_text()) != state: + raise + if hashlib.sha256((data_dir / "knowledge/index.json").read_bytes()).hexdigest() != state["checkpoint_sha256"]: + raise + (data_dir / PENDING_FILE).unlink() + fsync_directory(data_dir / "knowledge") + return + if actual != old_graph: + raise + for name in FILES: + if name in originals: + replace_file(stage / "previous" / name, data_dir / name) + else: + (data_dir / name).unlink(missing_ok=True) + fsync_directory(data_dir / "knowledge") + if not had_pending: + (data_dir / PENDING_FILE).unlink() + fsync_directory(data_dir / "knowledge") + raise + (data_dir / PENDING_FILE).unlink() + fsync_directory(data_dir / "knowledge") diff --git a/tools/knowledge/query.py b/tools/knowledge/query.py index 9fec79d..90722d5 100644 --- a/tools/knowledge/query.py +++ b/tools/knowledge/query.py @@ -974,7 +974,10 @@ def deliver_content(question: str, *, engine: str, limit: int, db_file, complete from tools.knowledge.sources import source_lifecycle_notes from tools.rag.common import ZPWIKI_ROOT - notes = source_lifecycle_notes(ZPWIKI_ROOT, index["index_version"].get("source_sha256")) + # Current checkpoints already carry lifecycle evidence. A changed local + # clone must never supplement a published generation with newer notes. + notes = ([] if index.get("academic_schema") == 1 else + source_lifecycle_notes(ZPWIKI_ROOT, index["index_version"].get("source_sha256"))) existing = {(row["source_path"], row.get("work_label"), row["text"]) for row in units} units = [ *units, diff --git a/tools/knowledge/store.py b/tools/knowledge/store.py index 0ba0028..f8ec1e2 100644 --- a/tools/knowledge/store.py +++ b/tools/knowledge/store.py @@ -11,8 +11,26 @@ from tools.answers.context_policy import normalized SCOPE = "zpwiki-content-v1" -def write_graph(index: dict) -> None: - settings = get_neo4j_settings() +def create_constraints(session) -> None: + for name, label, field in [ + ("evidence", "KnowledgeEvidence", "id"), + ("concept", "KnowledgeConcept", "id"), + ("person", "KnowledgePerson", "source_path"), + ("work", "KnowledgeWork", "id"), + ("advisor", "KnowledgeAdvisor", "id"), + ]: + session.run( + f"CREATE CONSTRAINT zp_content_{name} IF NOT EXISTS " + f"FOR (n:{label}) REQUIRE (n.graph_scope,n.{field}) IS UNIQUE" + ).consume() + session.run( + "CREATE CONSTRAINT zp_content_state IF NOT EXISTS " + "FOR (n:KnowledgeState) REQUIRE n.scope IS UNIQUE" + ).consume() + + +def write_graph_transaction(tx, index: dict, *, checkpoint_sha256: str | None = None) -> None: + """The caller owns the transaction and filesystem publication barrier.""" rows = [] for unit in index["units"]: rows.append( @@ -24,162 +42,155 @@ def write_graph(index: dict) -> None: "relations_json": json.dumps(unit["relations"], ensure_ascii=False), } ) - with ( - create_neo4j_driver(settings) as driver, - driver.session(database=settings.database) as session, - ): - driver.verify_connectivity() - for name, label, field in [ - ("evidence", "KnowledgeEvidence", "id"), - ("concept", "KnowledgeConcept", "id"), - ("person", "KnowledgePerson", "source_path"), - ("work", "KnowledgeWork", "id"), - ("advisor", "KnowledgeAdvisor", "id"), - ]: - session.run( - f"CREATE CONSTRAINT zp_content_{name} IF NOT EXISTS " - f"FOR (n:{label}) REQUIRE (n.graph_scope,n.{field}) IS UNIQUE" - ).consume() - session.run( - "CREATE CONSTRAINT zp_content_state IF NOT EXISTS " - "FOR (n:KnowledgeState) REQUIRE n.scope IS UNIQUE" + tx.run( + "MATCH (n:KnowledgeRelation {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE + ).consume() + tx.run( + "MATCH (n:KnowledgeEvidence {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE + ).consume() + tx.run( + "MATCH (n:KnowledgeConcept {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE + ).consume() + tx.run( + "MATCH (n:KnowledgePerson {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE + ).consume() + evidence_rows = [ + { + "evidence": {k: v for k, v in row.items() if k != "concepts"} + | {"graph_scope": SCOPE}, + "concepts": row["concepts"], + } + for row in rows + ] + tx.run( + """UNWIND $rows AS row + CREATE (e:KnowledgeEvidence) SET e=row.evidence + MERGE (p:KnowledgePerson {source_path:row.evidence.source_path,graph_scope:$scope}) + SET p.name=row.evidence.person + MERGE (p)-[:HAS_EVIDENCE]->(e) + FOREACH (concept IN row.concepts | + MERGE (c:KnowledgeConcept {id:concept.id,graph_scope:$scope}) + SET c.name=concept.name MERGE (e)-[:ABOUT]->(c))""", + rows=evidence_rows, + scope=SCOPE, + ).consume() + relations = [ + { + "left": identifier(normalized(r["subject"])), + "right": identifier(normalized(r["object"])), + "evidence": row["id"], + "predicate": r["predicate"], + } + for row in rows + for r in json.loads(row["relations_json"]) + ] + tx.run( + """UNWIND $rows AS row + MATCH (a:KnowledgeConcept {id:row.left,graph_scope:$scope}), + (b:KnowledgeConcept {id:row.right,graph_scope:$scope}), + (e:KnowledgeEvidence {id:row.evidence,graph_scope:$scope}) + CREATE (r:KnowledgeRelation {graph_scope:$scope,predicate:row.predicate,origin:'llm'}) + CREATE (r)-[:SUBJECT]->(a), (r)-[:OBJECT]->(b), (r)-[:EVIDENCED_BY]->(e)""", + rows=relations, + scope=SCOPE, + ).consume() + from tools.knowledge.sources import advisor_name + tx.run( + "MATCH (n:KnowledgeWork {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE + ).consume() + tx.run( + "MATCH (n:KnowledgeAdvisor {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE + ).consume() + works = [ + { + "id": identifier(u["source_path"], u["work_label"]), + **{ + k: u.get(k) for k in ("source_path", "work_label", "work_type", "work_year") + }, + "evidence": u["id"], + } + for u in index["units"] + if u["kind"] == "work_record" + ] + tx.run( + """UNWIND $rows AS row + MATCH (p:KnowledgePerson {source_path:row.source_path,graph_scope:$scope}), + (e:KnowledgeEvidence {id:row.evidence,graph_scope:$scope}) + MERGE (w:KnowledgeWork {id:row.id,graph_scope:$scope}) + SET w.label=row.work_label,w.type=row.work_type,w.year=row.work_year + MERGE (p)-[:HAS_WORK]->(w) MERGE (w)-[:EVIDENCED_BY]->(e)""", + rows=works, + scope=SCOPE, + ).consume() + for role, edge in [("supervisor", "SUPERVISED_BY"), ("consultant", "CONSULTED_BY")]: + advisors = [ + { + "work": identifier(u["source_path"], u["work_label"]), + "id": identifier(normalized(name)), + "name": name, + "evidence": u["id"], + } + for u in index["units"] + if u.get("work_label") and (name := advisor_name(u, role)) + ] + tx.run( + f"""UNWIND $rows AS row + MATCH (w:KnowledgeWork {{id:row.work,graph_scope:$scope}}), + (e:KnowledgeEvidence {{id:row.evidence,graph_scope:$scope}}) + MERGE (a:KnowledgeAdvisor {{id:row.id,graph_scope:$scope}}) SET a.name=row.name + MERGE (w)-[:{edge}]->(a) MERGE (e)-[:DOCUMENTS_ROLE]->(a)""", + rows=advisors, + scope=SCOPE, ).consume() - with session.begin_transaction() as tx: - tx.run( - "MATCH (n:KnowledgeRelation {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE - ).consume() - tx.run( - "MATCH (n:KnowledgeEvidence {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE - ).consume() - tx.run( - "MATCH (n:KnowledgeConcept {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE - ).consume() - tx.run( - "MATCH (n:KnowledgePerson {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE - ).consume() - # Copy only scalar evidence properties. Dynamic labels/Cypher from - # the model are never executed; all relationships have fixed types. - evidence_rows = [ - { - "evidence": {k: v for k, v in row.items() if k != "concepts"} - | {"graph_scope": SCOPE}, - "concepts": row["concepts"], - } - for row in rows - ] - tx.run( - """UNWIND $rows AS row - CREATE (e:KnowledgeEvidence) SET e=row.evidence - MERGE (p:KnowledgePerson {source_path:row.evidence.source_path,graph_scope:$scope}) - SET p.name=row.evidence.person - MERGE (p)-[:HAS_EVIDENCE]->(e) - FOREACH (concept IN row.concepts | - MERGE (c:KnowledgeConcept {id:concept.id,graph_scope:$scope}) - SET c.name=concept.name MERGE (e)-[:ABOUT]->(c))""", - rows=evidence_rows, - scope=SCOPE, - ).consume() - relations = [ - { - "left": identifier(normalized(r["subject"])), - "right": identifier(normalized(r["object"])), - "evidence": row["id"], - "predicate": r["predicate"], - } - for row in rows - for r in json.loads(row["relations_json"]) - ] - tx.run( - """UNWIND $rows AS row - MATCH (a:KnowledgeConcept {id:row.left,graph_scope:$scope}), - (b:KnowledgeConcept {id:row.right,graph_scope:$scope}), - (e:KnowledgeEvidence {id:row.evidence,graph_scope:$scope}) - CREATE (r:KnowledgeRelation {graph_scope:$scope,predicate:row.predicate,origin:'llm'}) - CREATE (r)-[:SUBJECT]->(a), (r)-[:OBJECT]->(b), (r)-[:EVIDENCED_BY]->(e)""", - rows=relations, - scope=SCOPE, - ).consume() - from tools.knowledge.sources import advisor_name + tx.run( + """MATCH (e:KnowledgeEvidence {graph_scope:$scope}) + WHERE e.meeting_id IS NOT NULL AND e.kind <> 'meeting' + MATCH (m:KnowledgeEvidence {graph_scope:$scope,kind:'meeting',meeting_id:e.meeting_id}) + MERGE (e)-[:RECORDED_AT]->(m)""", + scope=SCOPE, + ).consume() + tx.run( + """MATCH (p:KnowledgePerson {graph_scope:$scope})-[:HAS_WORK]->(w), + (p)-[:HAS_EVIDENCE]->(e) + WHERE e.work_label=w.label AND e.kind='meeting' + MERGE (w)-[:HAS_MEETING]->(e)""", + scope=SCOPE, + ).consume() + tx.run( + """MATCH (p:KnowledgePerson {graph_scope:$scope})-[:HAS_WORK]->(w), + (p)-[:HAS_EVIDENCE]->(e) + WHERE e.work_label=w.label AND e.completion_status IS NOT NULL + MERGE (w)-[:HAS_STATUS_RECORD]->(e)""", + scope=SCOPE, + ).consume() + tx.run( + "MERGE (s:KnowledgeState {scope:$scope}) SET s.version=$version, s.index_version_json=$index_version_json, s.checkpoint_sha256=$checkpoint_sha256", + scope=SCOPE, + version=index["version"], + index_version_json=json.dumps(index.get("index_version"), sort_keys=True), + checkpoint_sha256=checkpoint_sha256, + ).consume() - # Roles are explicit, source-backed edges attached to an exact work. - # Profile-level roles remain unassigned; they are never spread over - # every work on that profile. - tx.run( - "MATCH (n:KnowledgeWork {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE - ).consume() - tx.run( - "MATCH (n:KnowledgeAdvisor {graph_scope:$scope}) DETACH DELETE n", scope=SCOPE - ).consume() - works = [ - { - "id": identifier(u["source_path"], u["work_label"]), - **{ - k: u.get(k) for k in ("source_path", "work_label", "work_type", "work_year") - }, - "evidence": u["id"], - } - for u in index["units"] - if u["kind"] == "work_record" - ] - tx.run( - """UNWIND $rows AS row - MATCH (p:KnowledgePerson {source_path:row.source_path,graph_scope:$scope}), - (e:KnowledgeEvidence {id:row.evidence,graph_scope:$scope}) - MERGE (w:KnowledgeWork {id:row.id,graph_scope:$scope}) - SET w.label=row.work_label,w.type=row.work_type,w.year=row.work_year - MERGE (p)-[:HAS_WORK]->(w) MERGE (w)-[:EVIDENCED_BY]->(e)""", - rows=works, - scope=SCOPE, - ).consume() - for role, edge in [("supervisor", "SUPERVISED_BY"), ("consultant", "CONSULTED_BY")]: - advisors = [ - { - "work": identifier(u["source_path"], u["work_label"]), - "id": identifier(normalized(name)), - "name": name, - "evidence": u["id"], - } - for u in index["units"] - if u.get("work_label") and (name := advisor_name(u, role)) - ] - tx.run( - f"""UNWIND $rows AS row - MATCH (w:KnowledgeWork {{id:row.work,graph_scope:$scope}}), - (e:KnowledgeEvidence {{id:row.evidence,graph_scope:$scope}}) - MERGE (a:KnowledgeAdvisor {{id:row.id,graph_scope:$scope}}) SET a.name=row.name - MERGE (w)-[:{edge}]->(a) MERGE (e)-[:DOCUMENTS_ROLE]->(a)""", - rows=advisors, - scope=SCOPE, - ).consume() - tx.run( - """MATCH (e:KnowledgeEvidence {graph_scope:$scope}) - WHERE e.meeting_id IS NOT NULL AND e.kind <> 'meeting' - MATCH (m:KnowledgeEvidence {graph_scope:$scope,kind:'meeting',meeting_id:e.meeting_id}) - MERGE (e)-[:RECORDED_AT]->(m)""", - scope=SCOPE, - ).consume() - tx.run( - """MATCH (p:KnowledgePerson {graph_scope:$scope})-[:HAS_WORK]->(w), - (p)-[:HAS_EVIDENCE]->(e) - WHERE e.work_label=w.label AND e.kind='meeting' - MERGE (w)-[:HAS_MEETING]->(e)""", - scope=SCOPE, - ).consume() - tx.run( - """MATCH (p:KnowledgePerson {graph_scope:$scope})-[:HAS_WORK]->(w), - (p)-[:HAS_EVIDENCE]->(e) - WHERE e.work_label=w.label AND e.completion_status IS NOT NULL - MERGE (w)-[:HAS_STATUS_RECORD]->(e)""", - scope=SCOPE, - ).consume() - tx.run( - "MERGE (s:KnowledgeState {scope:$scope}) SET s.version=$version", - scope=SCOPE, - version=index["version"], - ).consume() + +def graph_state(session) -> dict | None: + record = session.run( + "MATCH (s:KnowledgeState {scope:$scope}) RETURN properties(s) AS state", scope=SCOPE + ).single() + return dict(record["state"]) if record else None + + +def write_graph(index: dict) -> None: + """Low-level graph import; the production builder publishes through a journal.""" + settings = get_neo4j_settings() + with create_neo4j_driver(settings) as driver, driver.session(database=settings.database) as session: + driver.verify_connectivity() + create_constraints(session) + with session.begin_transaction() as tx: + write_graph_transaction(tx, index) tx.commit() + def read_graph(index: dict) -> list[dict]: settings = get_neo4j_settings() with ( @@ -188,10 +199,13 @@ def read_graph(index: dict) -> list[dict]: ): with session.begin_transaction() as tx: state = tx.run( - "MATCH (s:KnowledgeState {scope:$scope}) RETURN s.version AS version", scope=SCOPE + "MATCH (s:KnowledgeState {scope:$scope}) RETURN s.version AS version, " + "s.checkpoint_sha256 AS checkpoint_sha256", scope=SCOPE ).single() if not state or state["version"] != index["version"]: raise RuntimeError("Content graph and extraction checkpoint differ.") + if index.get("_checkpoint_sha256") and state["checkpoint_sha256"] != index["_checkpoint_sha256"]: + raise RuntimeError("Content graph and checkpoint checksum differ.") rows = tx.run( """MATCH (p:KnowledgePerson {graph_scope:$scope})-[:HAS_EVIDENCE]->(e) OPTIONAL MATCH (e)-[:ABOUT]->(c:KnowledgeConcept) @@ -204,10 +218,22 @@ def read_graph(index: dict) -> list[dict]: def load_index(data_dir: Path, version: dict | None) -> dict | None: + from tools.knowledge.publication import read_state + import hashlib + + state = read_state(data_dir) path = data_dir / "knowledge/index.json" if not path.exists(): return None - index = json.loads(path.read_text()) + content = path.read_bytes() + index = json.loads(content) if not version or index.get("index_version") != version: raise RuntimeError("Content extraction belongs to another source snapshot. Rebuild it.") + if state is not None: + if not isinstance(state, dict) or state.get("schema") != 1: + raise RuntimeError("Content extraction publication state is invalid.") + if (state.get("version") != index.get("version") or state.get("index_version") != version + or hashlib.sha256(content).hexdigest() != state.get("checkpoint_sha256")): + raise RuntimeError("Content extraction checkpoint is invalid. Rebuild it.") + index["_checkpoint_sha256"] = state["checkpoint_sha256"] return index diff --git a/tools/rag/rebuild_index.py b/tools/rag/rebuild_index.py index 75e611f..6935d67 100644 --- a/tools/rag/rebuild_index.py +++ b/tools/rag/rebuild_index.py @@ -123,13 +123,6 @@ def rebuild_index( else: result = synchronize_indexes(root, target) - from tools.knowledge.build import refresh_existing - - if (target / "knowledge/index.json").exists(): - if progress: - progress("extracting_knowledge") - result["knowledge"] = refresh_existing(root, target) - duration = round( time.monotonic() - start, 2, diff --git a/tools/runtime/answer_health.py b/tools/runtime/answer_health.py new file mode 100644 index 0000000..6443b74 --- /dev/null +++ b/tools/runtime/answer_health.py @@ -0,0 +1,83 @@ +"""Read-only readiness of versioned content and the configured answer model.""" +from __future__ import annotations + +import hashlib +import json +from functools import lru_cache +from pathlib import Path + +from neo4j import Query + +from tools.answers.model_client import generation_status +from tools.graphrag.graph_db import create_neo4j_driver, get_neo4j_settings +from tools.knowledge.publication import read_state, state_matches_graph +from tools.knowledge.store import SCOPE +from tools.runtime.tool_health import PROBE_TIMEOUT_SECONDS, readiness +from tools.sync.index_state import IndexUnavailableError, index_snapshot + + +@lru_cache(maxsize=2) +def _checkpoint(path: str, inode: int, size: int, modified_ns: int, changed_ns: int): + content = Path(path).read_bytes() + index = json.loads(content) + return {"version": index.get("version"), "index_version": index.get("index_version"), + "sha256": hashlib.sha256(content).hexdigest(), "bytes": len(content)} + + +def content_readiness(db_file: Path, version: dict | None, *, graph_available: bool) -> dict: + try: + # tools_readiness released its lock: recheck the generation before reading. + with index_snapshot(db_file.parent) as current: + if not version or current != version: + return readiness(False, "snapshot_changed") + state = read_state(db_file.parent) + path = db_file.parent / "knowledge/index.json" + if not path.is_file(): + return readiness(False, "content_index_missing") + if not isinstance(state, dict) or state.get("schema") != 1: + return readiness(False, "content_publication_required") + st = path.stat() + actual = _checkpoint(str(path), st.st_ino, st.st_size, st.st_mtime_ns, st.st_ctime_ns) + if state.get("index_version") != version or actual["index_version"] != version: + return readiness(False, "content_version_mismatch") + if (actual["version"] != state.get("version") + or actual["sha256"] != state.get("checkpoint_sha256") + or actual["bytes"] != state.get("checkpoint_bytes")): + return readiness(False, "content_checkpoint_invalid") + details = {"index_version": version, "content_version": state["version"], + "checkpoint_sha256": actual["sha256"], "profiles": state.get("profiles"), + "units": state.get("units")} + graph = readiness(False, "content_graph_disabled") + if state.get("graph_ready"): + graph = _graph_readiness(state) if graph_available else readiness(False, "neo4j_unavailable") + return {**readiness(True, **details), "graph": graph} + except IndexUnavailableError: + return readiness(False, "content_publication_incomplete") + except (OSError, ValueError, TypeError, KeyError): + return readiness(False, "content_checkpoint_invalid") + + +def _graph_readiness(state): + try: + settings = get_neo4j_settings() + with create_neo4j_driver(settings, connection_timeout=PROBE_TIMEOUT_SECONDS, + connection_acquisition_timeout=PROBE_TIMEOUT_SECONDS, + connection_write_timeout=PROBE_TIMEOUT_SECONDS, + max_transaction_retry_time=0, max_connection_pool_size=1) as driver: + with driver.session(database=settings.database) as session: + row = session.run(Query( + "MATCH (s:KnowledgeState {scope:$scope}) RETURN properties(s) AS state", + timeout=PROBE_TIMEOUT_SECONDS), scope=SCOPE).single() + actual = dict(row["state"]) if row else None + if not state_matches_graph(state, actual): + return readiness(False, "content_graph_version_mismatch") + return readiness(True, content_version=state["version"]) + except Exception: + return readiness(False, "content_graph_unavailable") + + +def answer_capabilities(db_file, tools, version): + content = content_readiness(db_file, version, graph_available=tools["neo4j"]["ready"]) + generation = generation_status() + return {"text_search": tools["rag"], "metadata_graph": tools["graphrag"], + "knowledge": content, "generation": generation} diff --git a/tools/sync/sync_indexes.py b/tools/sync/sync_indexes.py index e72273b..a8aba61 100644 --- a/tools/sync/sync_indexes.py +++ b/tools/sync/sync_indexes.py @@ -16,7 +16,7 @@ import sqlite3 import subprocess import tempfile import uuid -from contextlib import closing +from contextlib import closing, ExitStack from datetime import datetime, timezone from pathlib import Path @@ -121,21 +121,28 @@ def replace_file(source: Path, target: Path) -> None: os.replace(source, target) -def publish_snapshot(data_dir: Path, stage: Path, version: dict, payload: dict, session) -> dict: +def publish_snapshot(data_dir: Path, stage: Path, version: dict, payload: dict, session, *, knowledge=None) -> dict: with publication_lock(data_dir, exclusive=True): + from tools.knowledge import publication as content_publication + from tools.knowledge import store as content_store + files = (*FILES, *content_publication.FILES) if knowledge is not None else FILES + content_state = (json.loads((stage / "knowledge/state.json").read_text()) + if knowledge is not None else None) had_pending = (data_dir / PENDING_FILE).exists() old_graph_version = graph_version(session) originals = [] backup = stage / "previous" backup.mkdir() - for name in FILES: + for name in files: current = data_dir / name if current.exists(): + (backup / name).parent.mkdir(parents=True, exist_ok=True) shutil.copy2(current, backup / name) with (backup / name).open("rb") as handle: os.fsync(handle.fileno()) originals.append(name) - fsync_directory(backup) + for directory in {backup, *((backup / name).parent for name in originals)}: + fsync_directory(directory) durable_json( data_dir / PENDING_FILE, { @@ -148,17 +155,22 @@ def publish_snapshot(data_dir: Path, stage: Path, version: dict, payload: dict, try: with session.begin_transaction() as tx: write_graph_transaction(tx, payload, version=version) + if knowledge is not None: + content_store.write_graph_transaction(tx, knowledge, + checkpoint_sha256=content_state["checkpoint_sha256"]) node_counts, relation_counts = graph_counts(tx) graph_summary = { "nodes": sum(n for _, n in node_counts), "relationships": sum(n for _, n in relation_counts), } - for name in FILES: + for name in files: # Flush staged contents before the durable directory switch. with (stage / name).open("rb") as handle: os.fsync(handle.fileno()) + (data_dir / name).parent.mkdir(parents=True, exist_ok=True) replace_file(stage / name, data_dir / name) - fsync_directory(data_dir) + for parent in { (data_dir / name).parent for name in files }: + fsync_directory(parent) commit_attempted = True tx.commit() except BaseException: @@ -173,22 +185,35 @@ def publish_snapshot(data_dir: Path, stage: Path, version: dict, payload: dict, current_graph_version == version and sqlite_version(data_dir / "zp_index.sqlite") == version ): + if knowledge is not None: + if not content_publication.state_matches_graph(content_state, content_store.graph_state(session)): + raise + if (json.loads((data_dir / "knowledge/state.json").read_text()) != content_state + or digest(data_dir / "knowledge/index.json") != content_state["checkpoint_sha256"]): + raise (data_dir / PENDING_FILE).unlink() + if knowledge is not None: + (data_dir / content_publication.PENDING_FILE).unlink(missing_ok=True) + fsync_directory(data_dir / 'knowledge') fsync_directory(data_dir) return graph_summary if current_graph_version != old_graph_version: raise # Unknown external writer; fail closed. - for name in FILES: + for name in files: if name in originals: os.replace(backup / name, data_dir / name) else: (data_dir / name).unlink(missing_ok=True) - fsync_directory(data_dir) + for directory in {(data_dir / name).parent for name in files}: + fsync_directory(directory) if not had_pending: (data_dir / PENDING_FILE).unlink() fsync_directory(data_dir) raise (data_dir / PENDING_FILE).unlink() + if knowledge is not None: + (data_dir / content_publication.PENDING_FILE).unlink(missing_ok=True) + fsync_directory(data_dir / 'knowledge') fsync_directory(data_dir) return graph_summary @@ -198,16 +223,24 @@ def synchronize_indexes(wiki_root: Path, data_dir: Path, *, progress=None) -> di stage = Path(tempfile.mkdtemp(prefix=".sync-stage-", dir=data_dir)) try: version, counts, payload = prepare_snapshot(wiki_root, stage) - if progress: - progress("publishing_indexes") - settings = get_neo4j_settings() - with ( - create_neo4j_driver(settings) as driver, - driver.session(database=settings.database) as session, - ): - driver.verify_connectivity() - create_constraints(session) - graph_summary = publish_snapshot(data_dir, stage, version, payload, session) + knowledge = None + knowledge_summary = None + with ExitStack() as locks: + if (data_dir / "knowledge/index.json").exists(): + from tools.rag.rebuild_index import acquire_reindex_lock + from tools.knowledge.build import prepare_index, configured_extractor + from tools.knowledge.publication import checkpoint_state + locks.enter_context(acquire_reindex_lock(data_dir / "knowledge/.build.lock")) + if progress: + progress("extracting_knowledge") + complete, identity = configured_extractor() + knowledge, knowledge_summary = prepare_index(stage / 'wiki', data_dir, complete, identity, + source_version=version) + checkpoint_state(stage, knowledge) + if git_revision(wiki_root) != version['git_revision'] or hashlib.sha256( + json.dumps(source_files(wiki_root),sort_keys=True).encode()).hexdigest() != version['source_sha256']: + raise RuntimeError("Wiki sa počas extrakcie zmenila; generácia nebola aktivovaná.") + graph_summary = _publish_prepared(data_dir, stage, version, payload, knowledge, progress) return { "counts": counts, "documents_scanned": counts["documents"], @@ -215,6 +248,7 @@ def synchronize_indexes(wiki_root: Path, data_dir: Path, *, progress=None) -> di "database_path": str(data_dir / "zp_index.sqlite"), "index_version": version, "graph_counts": graph_summary, + **({"knowledge": knowledge_summary} if knowledge_summary is not None else {}), } finally: # Keep recovery material when a crash/ambiguous commit is unresolved. @@ -227,3 +261,16 @@ def synchronize_indexes(wiki_root: Path, data_dir: Path, *, progress=None) -> di keep_stage = True if not keep_stage: shutil.rmtree(stage) + + +def _publish_prepared(data_dir, stage, version, payload, knowledge, progress): + if progress: + progress("publishing_indexes") + settings = get_neo4j_settings() + with create_neo4j_driver(settings) as driver, driver.session(database=settings.database) as session: + driver.verify_connectivity() + create_constraints(session) + if knowledge is not None: + from tools.knowledge.store import create_constraints as create_content_constraints + create_content_constraints(session) + return publish_snapshot(data_dir, stage, version, payload, session, knowledge=knowledge) diff --git a/tools/sync/sync_jobs.py b/tools/sync/sync_jobs.py index 619c4dd..cda614f 100644 --- a/tools/sync/sync_jobs.py +++ b/tools/sync/sync_jobs.py @@ -18,10 +18,13 @@ logger = logging.getLogger(__name__) class SyncJobs: - def __init__(self, data_dir: Path, *, runner: Callable | None = None): + def __init__(self, data_dir: Path, *, runner: Callable | None = None, periodic_interval_seconds: float = 0): self.data_dir = Path(data_dir) self.data_dir.mkdir(parents=True, exist_ok=True) self.path = self.data_dir / "sync_jobs.sqlite" + if periodic_interval_seconds < 0: + raise ValueError("Synchronization interval must not be negative") + self.periodic_interval_seconds = periodic_interval_seconds self.runner = runner or rebuild_index self.stopping = threading.Event() self.thread: threading.Thread | None = None @@ -34,6 +37,7 @@ class SyncJobs: request_count INTEGER NOT NULL DEFAULT 1, attempts INTEGER NOT NULL DEFAULT 0, available_at REAL NOT NULL DEFAULT 0, reason TEXT, result_json TEXT); CREATE INDEX IF NOT EXISTS jobs_status ON jobs(status, created_at); + CREATE TABLE IF NOT EXISTS schedule (id INTEGER PRIMARY KEY CHECK(id=1), next_due REAL NOT NULL); CREATE TABLE IF NOT EXISTS deliveries ( delivery_key TEXT PRIMARY KEY, job_id TEXT NOT NULL REFERENCES jobs(id)); """) @@ -64,45 +68,62 @@ class SyncJobs: now = time.time() with closing(self.connect()) as conn, conn: conn.execute("BEGIN IMMEDIATE") - if delivery_key: - duplicate = conn.execute( - "SELECT job_id FROM deliveries WHERE delivery_key=?", (delivery_key,) - ).fetchone() - if duplicate: - return { - "job": self.public( - conn.execute( - "SELECT * FROM jobs WHERE id=?", (duplicate["job_id"],) - ).fetchone() - ), - "coalesced": False, - "duplicate": True, - } - # Never merge a new request into a running job: its source snapshot - # may already exist. One queued follow-up absorbs further pushes. - queued = conn.execute( - "SELECT * FROM jobs WHERE status='queued' ORDER BY created_at LIMIT 1" + return self._enqueue(conn, now, pull_git, delivery_key, requested_revision) + + def _enqueue(self, conn, now, pull_git, delivery_key=None, requested_revision=None): + if delivery_key: + duplicate = conn.execute( + "SELECT job_id FROM deliveries WHERE delivery_key=?", (delivery_key,) ).fetchone() - if queued: - job_id = queued["id"] - conn.execute( - """UPDATE jobs SET pull_git=MAX(pull_git,?), updated_at=?, - requested_revision=COALESCE(?,requested_revision), request_count=request_count+1 - WHERE id=?""", - (int(pull_git), now, requested_revision, job_id), - ) - else: - job_id = uuid.uuid4().hex - conn.execute( - """INSERT INTO jobs(id,status,phase,pull_git,created_at,updated_at,requested_revision) - VALUES(?,'queued','queued',?,?,?,?)""", - (job_id, int(pull_git), now, now, requested_revision), - ) - if delivery_key: - conn.execute("INSERT INTO deliveries VALUES(?,?)", (delivery_key, job_id)) - job = self.public(conn.execute("SELECT * FROM jobs WHERE id=?", (job_id,)).fetchone()) + if duplicate: + return { + "job": self.public( + conn.execute( + "SELECT * FROM jobs WHERE id=?", (duplicate["job_id"],) + ).fetchone() + ), + "coalesced": False, + "duplicate": True, + } + # Never merge a new request into a running job: its source snapshot + # may already exist. One queued follow-up absorbs further pushes. + queued = conn.execute( + "SELECT * FROM jobs WHERE status='queued' ORDER BY created_at LIMIT 1" + ).fetchone() + if queued: + job_id = queued["id"] + conn.execute( + """UPDATE jobs SET pull_git=MAX(pull_git,?), updated_at=?, + requested_revision=COALESCE(?,requested_revision), request_count=request_count+1 + WHERE id=?""", + (int(pull_git), now, requested_revision, job_id), + ) + else: + job_id = uuid.uuid4().hex + conn.execute( + """INSERT INTO jobs(id,status,phase,pull_git,created_at,updated_at,requested_revision) + VALUES(?,'queued','queued',?,?,?,?)""", + (job_id, int(pull_git), now, now, requested_revision), + ) + if delivery_key: + conn.execute("INSERT INTO deliveries VALUES(?,?)", (delivery_key, job_id)) + job = self.public(conn.execute("SELECT * FROM jobs WHERE id=?", (job_id,)).fetchone()) return {"job": job, "coalesced": queued is not None, "duplicate": False} + def schedule_if_due(self) -> bool: + if not self.periodic_interval_seconds: + return False + now = time.time() + with closing(self.connect()) as conn, conn: + conn.execute("BEGIN IMMEDIATE") + row = conn.execute("SELECT next_due FROM schedule WHERE id=1").fetchone() + if row and row["next_due"] > now: + return False + self._enqueue(conn, now, True) + conn.execute("INSERT OR REPLACE INTO schedule VALUES(1,?)", + (now + self.periodic_interval_seconds,)) + return True + def get(self, job_id: str): with closing(self.connect()) as conn: return self.public(conn.execute("SELECT * FROM jobs WHERE id=?", (job_id,)).fetchone()) @@ -125,7 +146,11 @@ class SyncJobs: "SELECT * FROM jobs WHERE status='failed' ORDER BY finished_at DESC LIMIT 1" ).fetchone() ) - return {"active": active, "last_success": success, "last_failure": failure} + schedule = conn.execute("SELECT next_due FROM schedule WHERE id=1").fetchone() + return {"active": active, "last_success": success, "last_failure": failure, + "schedule": {"enabled": bool(self.periodic_interval_seconds), + "interval_seconds": self.periodic_interval_seconds, + "next_due": schedule["next_due"] if schedule else None}} def phase(self, job_id: str, phase: str): with closing(self.connect()) as conn, conn: @@ -208,6 +233,7 @@ class SyncJobs: def _loop(self): while not self.stopping.is_set(): try: + self.schedule_if_due() worked = self.run_once() except Exception: logger.exception("Chyba pracovníka synchronizačnej fronty.") diff --git a/tools/sync/wiki_git.py b/tools/sync/wiki_git.py index 59b70be..6f49245 100644 --- a/tools/sync/wiki_git.py +++ b/tools/sync/wiki_git.py @@ -82,6 +82,16 @@ def update_wiki(repo_path: Path) -> None: raise RuntimeError("Aktívna Git vetva nezodpovedá EXPECTED_GITEA_BRANCH") if git("status", "--porcelain", "--untracked-files=normal"): raise RuntimeError("Wiki obsahuje lokálne zmeny; automatická aktualizácia ich neprepíše") + if os.getenv("SYNC_UPSTREAM_FORK", "0") == "1": + if expected != "jp170na/zpwiki" or branch != "master": + raise RuntimeError("Upstream fork synchronization is restricted to jp170na/zpwiki master") + key_file = os.getenv("ZPWIKI_SYNC_KEY_FILE", "/run/secrets/zpwiki_ssh_key") + script = Path(__file__).resolve().parents[1] / "git_runner/sync_fork.sh" + result = subprocess.run(["bash", str(script), "--key-file", key_file], + env=environment, capture_output=True, text=True, timeout=GIT_TIMEOUT_SECONDS, + check=False) + if result.returncode: + raise RuntimeError("Fork synchronization failed; local indexes were not changed") git( "fetch", "--no-tags",