119 lines
9.3 KiB
Diff
119 lines
9.3 KiB
Diff
--- a/tools/answer_evaluation/context_policy.py
|
|
+++ b/tools/answer_evaluation/context_policy.py
|
|
@@ -32,17 +32,20 @@
|
|
|
|
|
|
def joined_candidates(facts: list[dict], intent: dict) -> list[dict]:
|
|
- """Join title/year and requested tag by person, retaining both real sources."""
|
|
- if not intent.get('mentions_title') or not (intent.get('mentions_keyword') or intent.get('keyword_phrase')):
|
|
+ """Join requested academic fields and tag by person, retaining real sources."""
|
|
+ needs_title = intent.get('mentions_title')
|
|
+ needs_year = intent.get('mentions_academic_year') or intent.get('years')
|
|
+ if not (needs_title or needs_year) or not (intent.get('mentions_keyword') or intent.get('keyword_phrase')):
|
|
return []
|
|
result=[]
|
|
for record in facts:
|
|
- if not record.get('title'):
|
|
+ if (needs_title and not record.get('title')) or (not needs_title and record.get('year') is None):
|
|
continue
|
|
for tag in facts:
|
|
if normalized(tag.get('person'))!=normalized(record.get('person')) or not tag.get('keyword'):
|
|
continue
|
|
- candidate={k:record[k] for k in ['person','title','year'] if record.get(k) is not None}
|
|
+ fields = ['person', 'title', 'year'] if needs_title else ['person', 'year']
|
|
+ candidate={k:record[k] for k in fields if record.get(k) is not None}
|
|
candidate['keyword']=tag['keyword']
|
|
candidate['evidence_documents']=sorted({p for p in [record.get('document_path'),tag.get('document_path')] if p})
|
|
candidate['source_urls']=sorted({u for u in [record.get('source_url'),tag.get('source_url')] if u})
|
|
--- a/tools/answer_evaluation/checked_generation.py
|
|
+++ b/tools/answer_evaluation/checked_generation.py
|
|
@@ -40,7 +40,7 @@
|
|
return result
|
|
feedback=('Odpoveď porušila pravidlá dodaného kontextu: '+', '.join(issues)+'. '
|
|
'Oprav jediný JSON objekt výhradne z pôvodných podkladov. Použi iba source_urls zo sources. '
|
|
- 'Typ práce nie je keyword. answer_candidates dokladajú spojenie názvu, roku a tagu cez tú istú osobu. '
|
|
+ 'Typ práce nie je keyword. answer_candidates dokladajú spojenie požadovaného názvu alebo roku a tagu cez tú istú osobu. '
|
|
'Pri neúplnom výstupe uveď: Výpis je neúplný.')
|
|
messages += [{'role':'assistant','content':text},{'role':'user','content':feedback}]
|
|
return result
|
|
--- a/tools/answer_evaluation/graph_answer_prompt.txt
|
|
+++ b/tools/answer_evaluation/graph_answer_prompt.txt
|
|
@@ -13,4 +13,4 @@
|
|
Vráť iba jediný platný JSON objekt bez úvodu, vysvetlení mimo JSON a bez Markdown ohraničenia. Reťazce ohraničuj ASCII dvojitými úvodzovkami. Vo vnútri answer názvy neobaľuj ďalšími ASCII dvojitými úvodzovkami; ak ich použiješ, musia byť escapované. Roky a count sú celé čísla. Nevypisuj čiarku za poslednou položkou.
|
|
Rešpektuj complete, truncated, truncation_reasons a index_version. Ak truncated=true alebo complete=false a uvádzaš výsledky, do answer vlož vetu „Výpis je neúplný.“ Neoznačuj výpis za úplný zoznam ani za všetky existujúce práce. Pri complete=null úplnosť nemožno zaručiť. Ak aggregation.complete=true, počet môže byť presný aj pri orezaných dôkazoch; vtedy uveď „Počet je úplný, dôkazy sú orezané.“ Inak z počtu zobrazených položiek neodvodzuj celkový počet.
|
|
Všetky URL v answer aj source_urls musia byť presne zo sources. Typ práce (napríklad diplomový projekt) nie je kľúčové slovo; keyword uveď iba ak ho dokladá fakt keyword pre tú istú person.
|
|
-answer_candidates sú doložené spojenia názvu, roku a tagu cez tú istú osobu, nie nové odhady. Ak je taký kandidát poskytnutý a status kontextu je ok, odpovedz podľa neho a neodmietaj ho len preto, že tag pochádza z dcérskeho dokumentu. V claims ponechaj person, title, year a keyword spolu v jednom objekte a cituj všetky jeho podklady.
|
|
+answer_candidates sú doložené spojenia požadovaného názvu alebo roku a tagu cez tú istú osobu, nie nové odhady. Ak je taký kandidát poskytnutý a status kontextu je ok, odpovedz podľa neho a neodmietaj ho len preto, že tag pochádza z dcérskeho dokumentu. V claims ponechaj jeho person, year, keyword a prípadne title spolu v jednom objekte a cituj všetky jeho podklady. Aj pri samotnej kombinácii tag + rok práce sa tag viaže na osobu, nemusí byť v dokumente obsahujúcom rok práce.
|
|
--- a/tools/graphrag/evaluation/check_openwebui.py
|
|
+++ b/tools/graphrag/evaluation/check_openwebui.py
|
|
@@ -82,6 +82,11 @@
|
|
|
|
|
|
def main():
|
|
+ # Import the common adapter here: it imports grade_claims from this module.
|
|
+ from tools.answer_evaluation.evaluate_graph_answers import graph_context, grade_answer, HERE
|
|
+ from tools.answer_evaluation.checked_generation import generate_checked_answer
|
|
+ from tools.graphrag.build_graphrag import graph_version
|
|
+
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--questions', type=Path, default=BASE / 'graphrag_questions_v4.json')
|
|
parser.add_argument('--freeze-manifest', type=Path, required=True)
|
|
@@ -100,8 +105,10 @@
|
|
selected.append(case); counts[case['category']] += 1
|
|
oracle = load_source_oracle(); manifest = json.loads(args.freeze_manifest.read_text())
|
|
settings = get_neo4j_settings(); rows = []
|
|
- payload = {'method': 'OpenWebUI API generation with locally supplied GraphRAG context; no autonomous tool routing.',
|
|
- 'model': args.model, 'selected_ids': [q['id'] for q in selected], 'system_prompt': SYSTEM,
|
|
+ prompt = (HERE / 'graph_answer_prompt.txt').read_text()
|
|
+ payload = {'method': 'Supplementary known-query check with common context guards; no autonomous tool routing.',
|
|
+ 'model': args.model, 'selected_ids': [q['id'] for q in selected], 'system_prompt': prompt,
|
|
+ 'corrective_request_limit': 1,
|
|
'limitations': ['Small stratified supplementary sample; not the primary 320-case accuracy.',
|
|
'Structured claims, source citations and binding checks; free prose requires separate review.'],
|
|
'questions': rows}
|
|
@@ -110,24 +117,24 @@
|
|
metadata = run_metadata(args.questions)
|
|
verify_freeze(manifest, metadata, graph_fingerprint(session), audit=args.audit, **manifest['parameters'])
|
|
payload['run_metadata'] = metadata
|
|
+ version = graph_version(session)
|
|
for case in selected:
|
|
started = time.perf_counter()
|
|
result = search_graph_with_session(session, case['question'], **manifest['parameters'])
|
|
- context = {k: result[k] for k in ['status', 'reason', 'facts', 'documents', 'aggregation']}
|
|
+ result['index_version'] = version
|
|
+ context = graph_context(result)
|
|
row = {'id': case['id'], 'category': case['category'], 'question': case['question'],
|
|
'graph_context': context, 'strict': False}
|
|
try:
|
|
- response = request_json(OPENWEBUI_URL, method='POST', timeout=90, max_attempts=1,
|
|
- headers={'Authorization': f'Bearer {os.environ["OPENWEBUI_API_KEY"]}', 'Content-Type': 'application/json'},
|
|
- payload={'model': args.model, 'stream': False, 'response_format': {'type': 'json_object'},
|
|
- 'messages': [{'role': 'system', 'content': SYSTEM},
|
|
- {'role': 'user', 'content': json.dumps({'question': case['question'], 'graph_context': context}, ensure_ascii=False)}]})
|
|
- row['response'] = response
|
|
- text = response['choices'][0]['message']['content'].strip()
|
|
- if text.startswith('```'):
|
|
- text = text.split('\n', 1)[1].rsplit('```', 1)[0]
|
|
- answer = json.loads(text)
|
|
- row['answer'] = answer; row.update(grade_claims(case, answer, oracle))
|
|
+ def complete(messages):
|
|
+ return request_json(OPENWEBUI_URL, method='POST', timeout=90, max_attempts=1,
|
|
+ headers={'Authorization': f'Bearer {os.environ["OPENWEBUI_API_KEY"]}', 'Content-Type': 'application/json'},
|
|
+ payload={'model': args.model, 'stream': False, 'response_format': {'type': 'json_object'},
|
|
+ 'messages': messages})
|
|
+ row.update(generate_checked_answer(case['question'], context, prompt, complete))
|
|
+ if row.get('answer'):
|
|
+ row.update(grade_answer(case, row['answer'], oracle, context=context))
|
|
+ row['strict'] = bool(row['strict'] and row['accepted_for_delivery'])
|
|
except Exception as error:
|
|
row['error'] = f'{type(error).__name__}: {str(error)[:500]}'
|
|
row['latency_seconds'] = time.perf_counter() - started
|
|
@@ -137,6 +144,7 @@
|
|
args.output.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + '\n')
|
|
print(f'{case["id"]}: {"OK" if row["strict"] else "FAIL"} ({row["latency_seconds"]:.1f}s)', flush=True)
|
|
verify_freeze(manifest, run_metadata(args.questions), graph_fingerprint(session), audit=args.audit, **manifest['parameters'])
|
|
+ assert graph_version(session) == version, 'Index version changed during generation.'
|
|
print(json.dumps(payload['summary']))
|
|
|
|
|