import hashlib import hmac import json import threading import time from concurrent.futures import ThreadPoolExecutor import pytest from fastapi.testclient import TestClient import app.main as main from tools.sync.sync_jobs import SyncJobs from tools.rag.rebuild_index import acquire_reindex_lock def test_concurrent_enqueues_are_durable_and_coalesced(tmp_path): jobs = SyncJobs(tmp_path) with ThreadPoolExecutor(max_workers=8) as pool: accepted = list( pool.map( lambda n: jobs.enqueue( pull_git=n % 2 == 0, delivery_key=f"delivery-{n}", requested_revision=str(n) ), range(24), ) ) assert len({row["job"]["id"] for row in accepted}) == 1 job = SyncJobs(tmp_path).get(accepted[0]["job"]["id"]) assert job["request_count"] == 24 and job["pull_git"] duplicate = SyncJobs(tmp_path).enqueue(pull_git=True, delivery_key="delivery-1") assert duplicate["duplicate"] and duplicate["job"]["request_count"] == 24 def test_push_during_running_job_schedules_followup_and_workers_do_not_overlap(tmp_path): entered, release = threading.Event(), threading.Event() phases = [] def runner(**kw): kw["progress"]("building_indexes") phases.append("run") entered.set() assert release.wait(5) return {"index_version": {"version": str(len(phases))}} jobs = SyncJobs(tmp_path, runner=runner) first = jobs.enqueue(pull_git=True)["job"]["id"] worker = threading.Thread(target=jobs.run_once) worker.start() try: assert entered.wait(5) assert jobs.get(first)["phase"] == "building_indexes" second = jobs.enqueue(pull_git=True)["job"]["id"] third = jobs.enqueue(pull_git=True)["job"]["id"] assert second != first and third == second assert not SyncJobs(tmp_path, runner=runner).run_once() finally: release.set() worker.join(5) assert jobs.get(first)["status"] == "succeeded" assert jobs.run_once() and jobs.get(second)["status"] == "succeeded" assert len(phases) == 2 def test_interrupted_job_recovers_and_preserves_delivery_dedup(tmp_path): jobs = SyncJobs(tmp_path) job_id = jobs.enqueue(pull_git=True, delivery_key="one")["job"]["id"] with jobs.connect() as conn: conn.execute("UPDATE jobs SET status='running',phase='publishing_indexes'") recovered = SyncJobs(tmp_path, runner=lambda **kw: {"index_version": {"version": "recovered"}}) assert recovered.run_once() assert recovered.get(job_id)["status"] == "succeeded" assert recovered.enqueue(pull_git=True, delivery_key="one")["duplicate"] def test_cli_lock_defers_instead_of_losing_job(tmp_path): def runner(**kw): with acquire_reindex_lock(tmp_path / ".reindex.lock"): return {"counts": {"documents": 1}} jobs = SyncJobs(tmp_path, runner=runner) job_id = jobs.enqueue(pull_git=False)["job"]["id"] with acquire_reindex_lock(tmp_path / ".reindex.lock"): assert jobs.run_once() job = jobs.get(job_id) assert job["status"] == "queued" and job["attempts"] == 0 with jobs.connect() as conn: conn.execute("UPDATE jobs SET available_at=0") jobs.run_once() assert jobs.get(job_id)["status"] == "succeeded" def test_background_worker_processes_and_stops(tmp_path): completed = threading.Event() def runner(**kw): completed.set() return {"counts": {"documents": 1}} jobs = SyncJobs(tmp_path, runner=runner) job_id = jobs.enqueue(pull_git=False)["job"]["id"] jobs.start() try: assert completed.wait(5) finally: jobs.stop() assert not jobs.thread.is_alive() and jobs.get(job_id)["status"] == "succeeded" @pytest.fixture def api(monkeypatch): monkeypatch.setattr(main, "embed_query", lambda _: None) with TestClient(main.app) as client: yield client def post_push(api, *, ref="refs/heads/master", delivery="one", **extra): body = json.dumps( {"repository": {"full_name": "KEMT/zpwiki"}, "ref": ref, "after": "a" * 40, **extra} ).encode() signature = hmac.new(("w" * 64).encode(), body, hashlib.sha256).hexdigest() return api.post( "/webhook/gitea", content=body, headers={ "X-Gitea-Event": "push", "X-Gitea-Signature": signature, "X-Gitea-Delivery": delivery, }, ) def test_api_returns_before_slow_rebuild_and_rejects_unauthed_status(api, monkeypatch): import app.routes as routes def forbidden(**kw): pytest.fail("HTTP request must not execute the rebuild") monkeypatch.setattr(main, "rebuild_index", forbidden) response = post_push(api) assert response.status_code == 202 assert api.get(response.json()["status_url"]).status_code == 401 assert api.get("/sync/status").status_code == 401 assert ( api.get(response.json()["status_url"], headers={"X-API-Key": "s" * 64}).json()["status"] == "queued" ) assert api.get("/sync/jobs/missing", headers={"X-API-Key": "s" * 64}).status_code == 404 def test_api_deduplicates_delivery_and_enables_git_pull(api, monkeypatch): monkeypatch.setenv("WEBHOOK_PULL_GIT", "true") first = post_push(api) repeated = post_push(api) assert first.json()["job_id"] == repeated.json()["job_id"] assert repeated.json()["duplicate"] and first.json()["pull_git"] assert main.app.state.sync_jobs.get(first.json()["job_id"])["requested_revision"] == "a" * 40 @pytest.mark.parametrize( "extra,reason", [ ({"ref": "refs/heads/other"}, "unexpected_branch"), ({"deleted": True}, "deleted_branch"), ({"after": "0" * 40}, "deleted_branch"), ], ) def test_other_branches_and_deletions_do_not_queue(api, extra, reason): response = post_push(api, **extra) assert response.status_code == 202 and response.json()["reason"] == reason assert main.app.state.sync_jobs.overview()["active"] == [] @pytest.mark.parametrize( "extra", [{"ref": None}, {"ref": "refs/tags/v1"}, {"after": "not-a-commit"}] ) def test_malformed_push_is_rejected(api, extra): assert post_push(api, **extra).status_code == 400 assert main.app.state.sync_jobs.overview()["active"] == [] def test_daily_schedule_is_durable_and_duplicate_workers_do_not_enqueue_twice(tmp_path, monkeypatch): monkeypatch.setattr(time, "time", lambda: 1000) jobs = SyncJobs(tmp_path, periodic_interval_seconds=86400) assert jobs.schedule_if_due() restarted = SyncJobs(tmp_path, periodic_interval_seconds=86400) assert not restarted.schedule_if_due() assert len(jobs.overview()["active"]) == 1 assert jobs.overview()["schedule"]["next_due"] == 87400 # Manual webhook gets the same queued job, without losing its Git update. assert jobs.enqueue(pull_git=False)["coalesced"] assert jobs.overview()["active"][0]["pull_git"] monkeypatch.setattr(time, "time", lambda: 87401) assert restarted.schedule_if_due() assert len(jobs.overview()["active"]) == 1 assert not SyncJobs(tmp_path).schedule_if_due()