57 lines
2.8 KiB
Python
57 lines
2.8 KiB
Python
import sys
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from scripts.evaluation.ragas_data import aggregate, checked_score, sample_from_answer, skip_reason
|
|
|
|
|
|
class RagasDataTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.case = {"question": "Question", "reference_answer": "Gold only", "expected": "answer_from_evidence"}
|
|
self.row = {"status": "ok", "mode": "mix", "response": {
|
|
"response": "Answer", "references": [{"content": ["Actual chunk", "Actual chunk"]}]}}
|
|
|
|
def test_reference_never_leaks_into_retrieved_context(self):
|
|
sample = sample_from_answer(self.row, self.case)
|
|
self.assertEqual(sample["retrieved_contexts"], ["Actual chunk"])
|
|
self.assertNotIn(self.case["reference_answer"], sample["retrieved_contexts"])
|
|
|
|
def test_bypass_context_metrics_are_not_zero(self):
|
|
self.row["mode"] = "bypass"
|
|
sample = sample_from_answer(self.row, self.case)
|
|
self.assertEqual(skip_reason(self.row,self.case,"faithfulness",sample),"no_retrieval_in_baseline")
|
|
self.assertIsNone(skip_reason(self.row,self.case,"answer_correctness",sample))
|
|
|
|
def test_abstention_not_penalized_by_answerable_metrics(self):
|
|
self.case["expected"] = "abstain_or_request_details"
|
|
self.assertEqual(skip_reason(self.row,self.case,"answer_relevancy",{}),"abstention_scenario_evaluated_separately")
|
|
|
|
def test_errors_not_included_in_score_mean(self):
|
|
rows=[{"case_id":"a","mode":"mix","metric":"faithfulness","status":"error"},
|
|
{"case_id":"b","mode":"mix","metric":"faithfulness","status":"ok","value":0.8}]
|
|
result=aggregate(rows)["mix"]["faithfulness"]
|
|
self.assertEqual(result["mean"],0.8)
|
|
self.assertEqual(result["n_error"],1)
|
|
|
|
def test_resume_does_not_double_count(self):
|
|
row={"case_id":"a","mode":"mix","metric":"faithfulness","status":"ok","value":0.8}
|
|
self.assertEqual(aggregate([row,row])["mix"]["faithfulness"]["n_ok"],1)
|
|
|
|
def test_nonfinite_scores_are_rejected(self):
|
|
with self.assertRaises(ValueError): checked_score("faithfulness",float("nan"))
|
|
self.assertEqual(checked_score("answer_relevancy",-0.1),-0.1)
|
|
|
|
def test_undefined_metric_is_not_a_zero_score(self):
|
|
rows=[{"case_id":"a","mode":"mix","metric":"faithfulness","status":"undefined","value":None}]
|
|
result=aggregate(rows)["mix"]["faithfulness"]
|
|
self.assertIsNone(result["mean"])
|
|
self.assertEqual(result["n_undefined"],1)
|
|
|
|
def test_string_context_cannot_be_silently_split_into_characters(self):
|
|
self.row["response"]["references"][0]["content"]="not a list"
|
|
with self.assertRaises(ValueError): sample_from_answer(self.row,self.case)
|
|
|
|
|
|
if __name__ == "__main__": unittest.main()
|