196 lines
7.0 KiB
Python
196 lines
7.0 KiB
Python
import hashlib
|
|
import hmac
|
|
import json
|
|
import threading
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
import app.main as main
|
|
from tools.sync.sync_jobs import SyncJobs
|
|
from tools.rag.rebuild_index import acquire_reindex_lock
|
|
|
|
|
|
def test_concurrent_enqueues_are_durable_and_coalesced(tmp_path):
|
|
jobs = SyncJobs(tmp_path)
|
|
with ThreadPoolExecutor(max_workers=8) as pool:
|
|
accepted = list(
|
|
pool.map(
|
|
lambda n: jobs.enqueue(
|
|
pull_git=n % 2 == 0, delivery_key=f"delivery-{n}", requested_revision=str(n)
|
|
),
|
|
range(24),
|
|
)
|
|
)
|
|
assert len({row["job"]["id"] for row in accepted}) == 1
|
|
job = SyncJobs(tmp_path).get(accepted[0]["job"]["id"])
|
|
assert job["request_count"] == 24 and job["pull_git"]
|
|
duplicate = SyncJobs(tmp_path).enqueue(pull_git=True, delivery_key="delivery-1")
|
|
assert duplicate["duplicate"] and duplicate["job"]["request_count"] == 24
|
|
|
|
|
|
def test_push_during_running_job_schedules_followup_and_workers_do_not_overlap(tmp_path):
|
|
entered, release = threading.Event(), threading.Event()
|
|
phases = []
|
|
|
|
def runner(**kw):
|
|
kw["progress"]("building_indexes")
|
|
phases.append("run")
|
|
entered.set()
|
|
assert release.wait(5)
|
|
return {"index_version": {"version": str(len(phases))}}
|
|
|
|
jobs = SyncJobs(tmp_path, runner=runner)
|
|
first = jobs.enqueue(pull_git=True)["job"]["id"]
|
|
worker = threading.Thread(target=jobs.run_once)
|
|
worker.start()
|
|
try:
|
|
assert entered.wait(5)
|
|
assert jobs.get(first)["phase"] == "building_indexes"
|
|
second = jobs.enqueue(pull_git=True)["job"]["id"]
|
|
third = jobs.enqueue(pull_git=True)["job"]["id"]
|
|
assert second != first and third == second
|
|
assert not SyncJobs(tmp_path, runner=runner).run_once()
|
|
finally:
|
|
release.set()
|
|
worker.join(5)
|
|
assert jobs.get(first)["status"] == "succeeded"
|
|
assert jobs.run_once() and jobs.get(second)["status"] == "succeeded"
|
|
assert len(phases) == 2
|
|
|
|
|
|
def test_interrupted_job_recovers_and_preserves_delivery_dedup(tmp_path):
|
|
jobs = SyncJobs(tmp_path)
|
|
job_id = jobs.enqueue(pull_git=True, delivery_key="one")["job"]["id"]
|
|
with jobs.connect() as conn:
|
|
conn.execute("UPDATE jobs SET status='running',phase='publishing_indexes'")
|
|
recovered = SyncJobs(tmp_path, runner=lambda **kw: {"index_version": {"version": "recovered"}})
|
|
assert recovered.run_once()
|
|
assert recovered.get(job_id)["status"] == "succeeded"
|
|
assert recovered.enqueue(pull_git=True, delivery_key="one")["duplicate"]
|
|
|
|
|
|
def test_cli_lock_defers_instead_of_losing_job(tmp_path):
|
|
def runner(**kw):
|
|
with acquire_reindex_lock(tmp_path / ".reindex.lock"):
|
|
return {"counts": {"documents": 1}}
|
|
|
|
jobs = SyncJobs(tmp_path, runner=runner)
|
|
job_id = jobs.enqueue(pull_git=False)["job"]["id"]
|
|
with acquire_reindex_lock(tmp_path / ".reindex.lock"):
|
|
assert jobs.run_once()
|
|
job = jobs.get(job_id)
|
|
assert job["status"] == "queued" and job["attempts"] == 0
|
|
with jobs.connect() as conn:
|
|
conn.execute("UPDATE jobs SET available_at=0")
|
|
jobs.run_once()
|
|
assert jobs.get(job_id)["status"] == "succeeded"
|
|
|
|
|
|
def test_background_worker_processes_and_stops(tmp_path):
|
|
completed = threading.Event()
|
|
|
|
def runner(**kw):
|
|
completed.set()
|
|
return {"counts": {"documents": 1}}
|
|
|
|
jobs = SyncJobs(tmp_path, runner=runner)
|
|
job_id = jobs.enqueue(pull_git=False)["job"]["id"]
|
|
jobs.start()
|
|
try:
|
|
assert completed.wait(5)
|
|
finally:
|
|
jobs.stop()
|
|
assert not jobs.thread.is_alive() and jobs.get(job_id)["status"] == "succeeded"
|
|
|
|
|
|
@pytest.fixture
|
|
def api(monkeypatch):
|
|
monkeypatch.setattr(main, "embed_query", lambda _: None)
|
|
with TestClient(main.app) as client:
|
|
yield client
|
|
|
|
|
|
def post_push(api, *, ref="refs/heads/master", delivery="one", **extra):
|
|
body = json.dumps(
|
|
{"repository": {"full_name": "KEMT/zpwiki"}, "ref": ref, "after": "a" * 40, **extra}
|
|
).encode()
|
|
signature = hmac.new(("w" * 64).encode(), body, hashlib.sha256).hexdigest()
|
|
return api.post(
|
|
"/webhook/gitea",
|
|
content=body,
|
|
headers={
|
|
"X-Gitea-Event": "push",
|
|
"X-Gitea-Signature": signature,
|
|
"X-Gitea-Delivery": delivery,
|
|
},
|
|
)
|
|
|
|
|
|
def test_api_returns_before_slow_rebuild_and_rejects_unauthed_status(api, monkeypatch):
|
|
import app.routes as routes
|
|
|
|
def forbidden(**kw):
|
|
pytest.fail("HTTP request must not execute the rebuild")
|
|
|
|
monkeypatch.setattr(main, "rebuild_index", forbidden)
|
|
response = post_push(api)
|
|
assert response.status_code == 202
|
|
assert api.get(response.json()["status_url"]).status_code == 401
|
|
assert api.get("/sync/status").status_code == 401
|
|
assert (
|
|
api.get(response.json()["status_url"], headers={"X-API-Key": "s" * 64}).json()["status"]
|
|
== "queued"
|
|
)
|
|
assert api.get("/sync/jobs/missing", headers={"X-API-Key": "s" * 64}).status_code == 404
|
|
|
|
|
|
def test_api_deduplicates_delivery_and_enables_git_pull(api, monkeypatch):
|
|
monkeypatch.setenv("WEBHOOK_PULL_GIT", "true")
|
|
first = post_push(api)
|
|
repeated = post_push(api)
|
|
assert first.json()["job_id"] == repeated.json()["job_id"]
|
|
assert repeated.json()["duplicate"] and first.json()["pull_git"]
|
|
assert main.app.state.sync_jobs.get(first.json()["job_id"])["requested_revision"] == "a" * 40
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"extra,reason",
|
|
[
|
|
({"ref": "refs/heads/other"}, "unexpected_branch"),
|
|
({"deleted": True}, "deleted_branch"),
|
|
({"after": "0" * 40}, "deleted_branch"),
|
|
],
|
|
)
|
|
def test_other_branches_and_deletions_do_not_queue(api, extra, reason):
|
|
response = post_push(api, **extra)
|
|
assert response.status_code == 202 and response.json()["reason"] == reason
|
|
assert main.app.state.sync_jobs.overview()["active"] == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"extra", [{"ref": None}, {"ref": "refs/tags/v1"}, {"after": "not-a-commit"}]
|
|
)
|
|
def test_malformed_push_is_rejected(api, extra):
|
|
assert post_push(api, **extra).status_code == 400
|
|
assert main.app.state.sync_jobs.overview()["active"] == []
|
|
|
|
|
|
def test_daily_schedule_is_durable_and_duplicate_workers_do_not_enqueue_twice(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(time, "time", lambda: 1000)
|
|
jobs = SyncJobs(tmp_path, periodic_interval_seconds=86400)
|
|
assert jobs.schedule_if_due()
|
|
restarted = SyncJobs(tmp_path, periodic_interval_seconds=86400)
|
|
assert not restarted.schedule_if_due()
|
|
assert len(jobs.overview()["active"]) == 1
|
|
assert jobs.overview()["schedule"]["next_due"] == 87400
|
|
# Manual webhook gets the same queued job, without losing its Git update.
|
|
assert jobs.enqueue(pull_git=False)["coalesced"]
|
|
assert jobs.overview()["active"][0]["pull_git"]
|
|
monkeypatch.setattr(time, "time", lambda: 87401)
|
|
assert restarted.schedule_if_due()
|
|
assert len(jobs.overview()["active"]) == 1
|
|
assert not SyncJobs(tmp_path).schedule_if_due()
|