dp-zp-agent/test/test_sync_jobs.py

import hashlib
import hmac
import json
import threading
import time
from concurrent.futures import ThreadPoolExecutor

import pytest
from fastapi.testclient import TestClient

import app.main as main
from tools.sync.sync_jobs import SyncJobs
from tools.rag.rebuild_index import acquire_reindex_lock


def test_concurrent_enqueues_are_durable_and_coalesced(tmp_path):
    jobs = SyncJobs(tmp_path)
    with ThreadPoolExecutor(max_workers=8) as pool:
        accepted = list(pool.map(lambda n: jobs.enqueue(pull_git=n % 2 == 0,
            delivery_key=f'delivery-{n}', requested_revision=str(n)), range(24)))
    assert len({row['job']['id'] for row in accepted}) == 1
    job = SyncJobs(tmp_path).get(accepted[0]['job']['id'])
    assert job['request_count'] == 24 and job['pull_git']
    duplicate = SyncJobs(tmp_path).enqueue(pull_git=True, delivery_key='delivery-1')
    assert duplicate['duplicate'] and duplicate['job']['request_count'] == 24


def test_push_during_running_job_schedules_followup_and_workers_do_not_overlap(tmp_path):
    entered, release = threading.Event(), threading.Event()
    phases = []
    def runner(**kw):
        kw['progress']('building_indexes')
        phases.append('run')
        entered.set()
        assert release.wait(5)
        return {'index_version': {'version': str(len(phases))}}
    jobs = SyncJobs(tmp_path, runner=runner)
    first = jobs.enqueue(pull_git=True)['job']['id']
    worker = threading.Thread(target=jobs.run_once)
    worker.start()
    try:
        assert entered.wait(5)
        assert jobs.get(first)['phase'] == 'building_indexes'
        second = jobs.enqueue(pull_git=True)['job']['id']
        third = jobs.enqueue(pull_git=True)['job']['id']
        assert second != first and third == second
        assert not SyncJobs(tmp_path, runner=runner).run_once()
    finally:
        release.set()
        worker.join(5)
    assert jobs.get(first)['status'] == 'succeeded'
    assert jobs.run_once() and jobs.get(second)['status'] == 'succeeded'
    assert len(phases) == 2


def test_interrupted_job_recovers_and_preserves_delivery_dedup(tmp_path):
    jobs = SyncJobs(tmp_path)
    job_id = jobs.enqueue(pull_git=True, delivery_key='one')['job']['id']
    with jobs.connect() as conn:
        conn.execute("UPDATE jobs SET status='running',phase='publishing_indexes'")
    recovered = SyncJobs(tmp_path, runner=lambda **kw: {'index_version': {'version': 'recovered'}})
    assert recovered.run_once()
    assert recovered.get(job_id)['status'] == 'succeeded'
    assert recovered.enqueue(pull_git=True, delivery_key='one')['duplicate']


def test_cli_lock_defers_instead_of_losing_job(tmp_path):
    def runner(**kw):
        with acquire_reindex_lock(tmp_path / '.reindex.lock'):
            return {'counts': {'documents': 1}}
    jobs = SyncJobs(tmp_path, runner=runner)
    job_id = jobs.enqueue(pull_git=False)['job']['id']
    with acquire_reindex_lock(tmp_path / '.reindex.lock'):
        assert jobs.run_once()
    job = jobs.get(job_id)
    assert job['status'] == 'queued' and job['attempts'] == 0
    with jobs.connect() as conn:
        conn.execute('UPDATE jobs SET available_at=0')
    jobs.run_once()
    assert jobs.get(job_id)['status'] == 'succeeded'


def test_background_worker_processes_and_stops(tmp_path):
    completed = threading.Event()
    def runner(**kw):
        completed.set()
        return {'counts': {'documents': 1}}
    jobs = SyncJobs(tmp_path, runner=runner)
    job_id = jobs.enqueue(pull_git=False)['job']['id']
    jobs.start()
    try:
        assert completed.wait(5)
    finally:
        jobs.stop()
    assert not jobs.thread.is_alive() and jobs.get(job_id)['status'] == 'succeeded'


@pytest.fixture
def api(monkeypatch):
    monkeypatch.setattr(main, 'embed_query', lambda _: None)
    with TestClient(main.app) as client:
        yield client


def post_push(api, *, ref='refs/heads/master', delivery='one', **extra):
    body = json.dumps({'repository': {'full_name': 'KEMT/zpwiki'}, 'ref': ref,
                       'after': 'a' * 40, **extra}).encode()
    signature = hmac.new(('w' * 64).encode(), body, hashlib.sha256).hexdigest()
    return api.post('/webhook/gitea', content=body, headers={
        'X-Gitea-Event': 'push', 'X-Gitea-Signature': signature, 'X-Gitea-Delivery': delivery})


def test_api_returns_before_slow_rebuild_and_rejects_unauthed_status(api, monkeypatch):
    import app.routes as routes
    def forbidden(**kw):
        pytest.fail('HTTP request must not execute the rebuild')
    monkeypatch.setattr(main, 'rebuild_index', forbidden)
    response = post_push(api)
    assert response.status_code == 202
    assert api.get(response.json()['status_url']).status_code == 401
    assert api.get('/sync/status').status_code == 401
    assert api.get(response.json()['status_url'], headers={'X-API-Key': 's' * 64}).json()['status'] == 'queued'
    assert api.get('/sync/jobs/missing', headers={'X-API-Key': 's' * 64}).status_code == 404


def test_api_deduplicates_delivery_and_enables_git_pull(api, monkeypatch):
    monkeypatch.setenv('WEBHOOK_PULL_GIT', 'true')
    first = post_push(api)
    repeated = post_push(api)
    assert first.json()['job_id'] == repeated.json()['job_id']
    assert repeated.json()['duplicate'] and first.json()['pull_git']
    assert main.app.state.sync_jobs.get(first.json()['job_id'])['requested_revision'] == 'a' * 40


@pytest.mark.parametrize('extra,reason', [
    ({'ref': 'refs/heads/other'}, 'unexpected_branch'),
    ({'deleted': True}, 'deleted_branch'), ({'after': '0' * 40}, 'deleted_branch')])
def test_other_branches_and_deletions_do_not_queue(api, extra, reason):
    response = post_push(api, **extra)
    assert response.status_code == 202 and response.json()['reason'] == reason
    assert main.app.state.sync_jobs.overview()['active'] == []


@pytest.mark.parametrize('extra', [{'ref': None}, {'ref': 'refs/tags/v1'}, {'after': 'not-a-commit'}])
def test_malformed_push_is_rejected(api, extra):
    assert post_push(api, **extra).status_code == 400
    assert main.app.state.sync_jobs.overview()['active'] == []