import hashlib import hmac import json import threading import time from concurrent.futures import ThreadPoolExecutor import pytest from fastapi.testclient import TestClient import app.main as main from tools.sync_jobs import SyncJobs from tools.rag.rebuild_index import acquire_reindex_lock def test_concurrent_enqueues_are_durable_and_coalesced(tmp_path): jobs = SyncJobs(tmp_path) with ThreadPoolExecutor(max_workers=8) as pool: accepted = list(pool.map(lambda n: jobs.enqueue(pull_git=n % 2 == 0, delivery_key=f'delivery-{n}', requested_revision=str(n)), range(24))) assert len({row['job']['id'] for row in accepted}) == 1 job = SyncJobs(tmp_path).get(accepted[0]['job']['id']) assert job['request_count'] == 24 and job['pull_git'] duplicate = SyncJobs(tmp_path).enqueue(pull_git=True, delivery_key='delivery-1') assert duplicate['duplicate'] and duplicate['job']['request_count'] == 24 def test_push_during_running_job_schedules_followup_and_workers_do_not_overlap(tmp_path): entered, release = threading.Event(), threading.Event() phases = [] def runner(**kw): kw['progress']('building_indexes') phases.append('run') entered.set() assert release.wait(5) return {'index_version': {'version': str(len(phases))}} jobs = SyncJobs(tmp_path, runner=runner) first = jobs.enqueue(pull_git=True)['job']['id'] worker = threading.Thread(target=jobs.run_once) worker.start() try: assert entered.wait(5) assert jobs.get(first)['phase'] == 'building_indexes' second = jobs.enqueue(pull_git=True)['job']['id'] third = jobs.enqueue(pull_git=True)['job']['id'] assert second != first and third == second assert not SyncJobs(tmp_path, runner=runner).run_once() finally: release.set() worker.join(5) assert jobs.get(first)['status'] == 'succeeded' assert jobs.run_once() and jobs.get(second)['status'] == 'succeeded' assert len(phases) == 2 def test_interrupted_job_recovers_and_preserves_delivery_dedup(tmp_path): jobs = SyncJobs(tmp_path) job_id = jobs.enqueue(pull_git=True, delivery_key='one')['job']['id'] with jobs.connect() as conn: conn.execute("UPDATE jobs SET status='running',phase='publishing_indexes'") recovered = SyncJobs(tmp_path, runner=lambda **kw: {'index_version': {'version': 'recovered'}}) assert recovered.run_once() assert recovered.get(job_id)['status'] == 'succeeded' assert recovered.enqueue(pull_git=True, delivery_key='one')['duplicate'] def test_cli_lock_defers_instead_of_losing_job(tmp_path): def runner(**kw): with acquire_reindex_lock(tmp_path / '.reindex.lock'): return {'counts': {'documents': 1}} jobs = SyncJobs(tmp_path, runner=runner) job_id = jobs.enqueue(pull_git=False)['job']['id'] with acquire_reindex_lock(tmp_path / '.reindex.lock'): assert jobs.run_once() job = jobs.get(job_id) assert job['status'] == 'queued' and job['attempts'] == 0 with jobs.connect() as conn: conn.execute('UPDATE jobs SET available_at=0') jobs.run_once() assert jobs.get(job_id)['status'] == 'succeeded' def test_background_worker_processes_and_stops(tmp_path): completed = threading.Event() def runner(**kw): completed.set() return {'counts': {'documents': 1}} jobs = SyncJobs(tmp_path, runner=runner) job_id = jobs.enqueue(pull_git=False)['job']['id'] jobs.start() try: assert completed.wait(5) finally: jobs.stop() assert not jobs.thread.is_alive() and jobs.get(job_id)['status'] == 'succeeded' @pytest.fixture def api(monkeypatch): monkeypatch.setattr(main, 'embed_query', lambda _: None) with TestClient(main.app) as client: yield client def post_push(api, *, ref='refs/heads/master', delivery='one', **extra): body = json.dumps({'repository': {'full_name': 'KEMT/zpwiki'}, 'ref': ref, 'after': 'a' * 40, **extra}).encode() signature = hmac.new(('w' * 64).encode(), body, hashlib.sha256).hexdigest() return api.post('/webhook/gitea', content=body, headers={ 'X-Gitea-Event': 'push', 'X-Gitea-Signature': signature, 'X-Gitea-Delivery': delivery}) def test_api_returns_before_slow_rebuild_and_rejects_unauthed_status(api, monkeypatch): import app.routes as routes def forbidden(**kw): pytest.fail('HTTP request must not execute the rebuild') monkeypatch.setattr(main, 'rebuild_index', forbidden) response = post_push(api) assert response.status_code == 202 assert api.get(response.json()['status_url']).status_code == 401 assert api.get('/sync/status').status_code == 401 assert api.get(response.json()['status_url'], headers={'X-API-Key': 's' * 64}).json()['status'] == 'queued' assert api.get('/sync/jobs/missing', headers={'X-API-Key': 's' * 64}).status_code == 404 def test_api_deduplicates_delivery_and_enables_git_pull(api, monkeypatch): monkeypatch.setenv('WEBHOOK_PULL_GIT', 'true') first = post_push(api) repeated = post_push(api) assert first.json()['job_id'] == repeated.json()['job_id'] assert repeated.json()['duplicate'] and first.json()['pull_git'] assert main.app.state.sync_jobs.get(first.json()['job_id'])['requested_revision'] == 'a' * 40 @pytest.mark.parametrize('extra,reason', [ ({'ref': 'refs/heads/other'}, 'unexpected_branch'), ({'deleted': True}, 'deleted_branch'), ({'after': '0' * 40}, 'deleted_branch')]) def test_other_branches_and_deletions_do_not_queue(api, extra, reason): response = post_push(api, **extra) assert response.status_code == 202 and response.json()['reason'] == reason assert main.app.state.sync_jobs.overview()['active'] == [] @pytest.mark.parametrize('extra', [{'ref': None}, {'ref': 'refs/tags/v1'}, {'after': 'not-a-commit'}]) def test_malformed_push_is_rejected(api, extra): assert post_push(api, **extra).status_code == 400 assert main.app.state.sync_jobs.overview()['active'] == []