119 lines
4.7 KiB
Python
119 lines
4.7 KiB
Python
"""Protect historical evidence and boundaries changed by project cleanup."""
|
|
|
|
import hashlib
|
|
import io
|
|
import json
|
|
from pathlib import Path
|
|
import sqlite3
|
|
import tarfile
|
|
|
|
import pytest
|
|
|
|
from tools.evaluation.archive import restore, verify
|
|
|
|
|
|
@pytest.fixture
|
|
def archived_experiment(tmp_path):
|
|
experiments = tmp_path / "source/experiments"
|
|
experiments.mkdir(parents=True)
|
|
payload = b'{"original":true}\n'
|
|
checksum = hashlib.sha256(payload).hexdigest()
|
|
archive = experiments / "runs.tar.gz"
|
|
with tarfile.open(archive, "w:gz") as stream:
|
|
info = tarfile.TarInfo("blobs/" + checksum)
|
|
info.size = len(payload)
|
|
stream.addfile(info, io.BytesIO(payload))
|
|
entries = [
|
|
{
|
|
"original_path": name,
|
|
"path": "experiments/" + name,
|
|
"location": "archive",
|
|
"member": "blobs/" + checksum,
|
|
"bytes": len(payload),
|
|
"sha256": checksum,
|
|
}
|
|
for name in ("a.json", "b.json")
|
|
]
|
|
catalog = {
|
|
"schema_version": 1,
|
|
"source_commit": "original-commit",
|
|
"archive": archive.name,
|
|
"archive_sha256": hashlib.sha256(archive.read_bytes()).hexdigest(),
|
|
"entries": entries,
|
|
}
|
|
(experiments / "catalog.json").write_text(json.dumps(catalog))
|
|
return experiments, catalog, payload
|
|
|
|
|
|
def test_all_original_experiment_bytes_are_preserved():
|
|
result = verify()
|
|
assert result["entries"] == 279
|
|
assert result["source_commit"] == "3d0dfa154b57846187cc18772987a96ea1c90c31"
|
|
|
|
|
|
def test_restore_refuses_collision_before_writing_anything(archived_experiment, tmp_path):
|
|
experiments, _, payload = archived_experiment
|
|
destination = tmp_path / "restore"
|
|
collision = destination / "experiments/b.json"
|
|
collision.parent.mkdir(parents=True)
|
|
collision.write_bytes(b"do not replace")
|
|
with pytest.raises(FileExistsError):
|
|
restore(destination, experiments=experiments)
|
|
assert not (destination / "experiments/a.json").exists()
|
|
assert collision.read_bytes() == b"do not replace"
|
|
collision.unlink()
|
|
restored = restore(destination, experiments=experiments)
|
|
assert len(restored) == 2 and all(p.read_bytes() == payload for p in restored)
|
|
|
|
|
|
def test_restore_rejects_path_escape_and_corrupted_archive(archived_experiment, tmp_path):
|
|
experiments, catalog, _ = archived_experiment
|
|
catalog["entries"][0]["path"] = "../escape.json"
|
|
(experiments / "catalog.json").write_text(json.dumps(catalog))
|
|
with pytest.raises(ValueError, match="Unsafe"):
|
|
restore(tmp_path / "restore", experiments=experiments)
|
|
assert not (tmp_path / "escape.json").exists()
|
|
(experiments / "runs.tar.gz").write_bytes(b"corrupt")
|
|
with pytest.raises(ValueError, match="checksum"):
|
|
verify(experiments)
|
|
|
|
|
|
def test_restore_rejects_symlink_escape(archived_experiment, tmp_path):
|
|
experiments, _, _ = archived_experiment
|
|
destination = tmp_path / "restore"
|
|
destination.mkdir()
|
|
(destination / "experiments").symlink_to(tmp_path / "outside")
|
|
with pytest.raises(ValueError, match="escapes"):
|
|
restore(destination, experiments=experiments)
|
|
|
|
|
|
def test_schema_consolidation_preserves_table_and_view_distinction():
|
|
from tools.rag import (
|
|
rag_document_expansion,
|
|
rag_query_evidence,
|
|
rag_supplemental_expansion,
|
|
rag_utils,
|
|
)
|
|
|
|
with sqlite3.connect(":memory:") as connection:
|
|
connection.execute("CREATE TABLE records (name TEXT, year INTEGER)")
|
|
connection.execute("CREATE VIEW record_view AS SELECT * FROM records")
|
|
for module in (rag_query_evidence, rag_supplemental_expansion, rag_utils):
|
|
assert module.sqlite_table_exists(connection, "record_view")
|
|
assert not module.sqlite_table_exists(connection, "absent")
|
|
for module in (rag_query_evidence, rag_supplemental_expansion):
|
|
assert module.sqlite_table_columns(connection, "record_view") == {"name", "year"}
|
|
assert module.sqlite_table_columns(connection, "absent") == set()
|
|
assert not rag_document_expansion.sqlite_table_exists(connection, "record_view")
|
|
assert rag_document_expansion.sqlite_table_exists(connection, "records")
|
|
|
|
|
|
def test_freeze_separates_runtime_and_evaluation_code():
|
|
from tools.graphrag.evaluation.reproducibility import content_fingerprints
|
|
|
|
fingerprints = content_fingerprints(Path("experiments/graphrag/graphrag_questions_v7.json"))
|
|
assert "tools/knowledge/query.py" in fingerprints["implementation"]
|
|
assert "tools/answers/context_policy.py" in fingerprints["implementation"]
|
|
assert "tools/evaluation/archive.py" in fingerprints["evaluation"]
|
|
assert not any("/tests/" in p or "/evaluation/" in p for p in fingerprints["implementation"])
|