#!/usr/bin/env python3 """Evaluate the trained adapter on the prepared eval or test split; report loss, F1 and exact match.""" from __future__ import annotations import argparse from collections import Counter, defaultdict import hashlib import json import math import os from pathlib import Path import subprocess import sys import unicodedata from train import ROOT, read_config, write_config, verify_data, verify_length_filter, run_logged def normalized_tokens(text: str) -> list[str]: text = unicodedata.normalize("NFKC", text).casefold() return "".join( " " if unicodedata.category(char).startswith("P") else char for char in text ).split() def score_pair(prediction: str, reference: str) -> dict: predicted = normalized_tokens(prediction) expected = normalized_tokens(reference) if not expected: return { "token_f1": None, "normalized_exact_match": None, "scored": False, "skip_reason": "reference_has_zero_normalized_tokens", "prediction_tokens": len(predicted), "reference_tokens": 0, } shared = sum((Counter(predicted) & Counter(expected)).values()) f1 = 200.0 * shared / (len(predicted) + len(expected)) return { "token_f1": f1, "normalized_exact_match": 100.0 if predicted == expected else 0.0, "scored": True, "skip_reason": None, "prediction_tokens": len(predicted), "reference_tokens": len(expected), } def read_jsonl(path: Path) -> list[dict]: result = [] with path.open("r", encoding="utf-8-sig") as handle: for line_number, line in enumerate(handle, 1): if not line.strip(): continue try: row = json.loads(line) except json.JSONDecodeError as exc: raise ValueError(f"{path}:{line_number}: invalid JSON: {exc.msg}") from exc if not isinstance(row, dict): raise ValueError(f"{path}:{line_number}: expected a JSON object") result.append(row) return result def required_text(row: dict, field: str, context: str) -> str: value = row.get(field) if not isinstance(value, str): raise ValueError(f"{context}: {field!r} must contain a string") return value def reference_text(row: dict, index: int) -> str: for field in ("output", "response", "label"): if field in row: return required_text(row, field, f"reference row {index}") raise ValueError(f"reference row {index}: missing output/response/label") def sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for block in iter(lambda: handle.read(1024 * 1024), b""): digest.update(block) return digest.hexdigest() def verify_training_dataset(adapter: Path, data_dir: Path) -> Path: run_dir = adapter.parent if adapter.name.startswith("checkpoint-") and not (run_dir / "run.json").is_file(): run_dir = run_dir.parent record_path = run_dir / "run.json" if not record_path.is_file(): raise ValueError(f"Missing training record: {record_path}. Keep the adapter with its original run files.") record = json.loads(record_path.read_text(encoding="utf-8")) if record.get("data_manifest_sha256") != sha256(data_dir / "manifest.json"): raise ValueError("The prepared dataset differs from this adapter's training data. " "Use the original dataset and its eval/test split recorded in run.json.") return run_dir def summarize(rows: list[dict]) -> dict: scored = [row for row in rows if row["scored"]] count = len(scored) return { "count_total": len(rows), "count_scored": count, "count_skipped_zero_token_references": len(rows) - count, "count_empty_predictions_raw": sum(not row["prediction"].strip() for row in rows), "count_zero_token_predictions": sum(row["prediction_tokens"] == 0 for row in rows), "count_nonempty_predictions_raw": sum(bool(row["prediction"].strip()) for row in rows), "count_truncated_or_modified_labels": sum( not row["label_matches_full_reference"] for row in rows ), "count_labels_different_before_normalization": sum( not row["label_matches_full_reference_raw"] for row in rows ), "token_f1": sum(row["token_f1"] for row in scored) / count if count else None, "normalized_exact_match": ( sum(row["normalized_exact_match"] for row in scored) / count if count else None ), "prediction_tokens_total": sum(row["prediction_tokens"] for row in rows), "reference_tokens_total": sum(row["reference_tokens"] for row in rows), } def score_files( predictions_path: Path, references_path: Path, output_dir: Path, metadata_path: Path | None = None, ) -> dict: predictions = read_jsonl(predictions_path) references = read_jsonl(references_path) if not references: raise ValueError("The reference file contains no examples") if len(predictions) != len(references): raise ValueError( f"Row count mismatch: {len(predictions)} predictions != {len(references)} references. " "No scores written; evaluate the complete split in the original order." ) metadata = read_jsonl(metadata_path) if metadata_path is not None else [{} for _ in references] if len(metadata) != len(references): raise ValueError( f"Row count mismatch: {len(metadata)} metadata rows != {len(references)} references" ) rows = [] groups = defaultdict(list) for index, (prediction_row, reference_row, meta) in enumerate( zip(predictions, references, metadata) ): prediction = required_text(prediction_row, "predict", f"prediction row {index}") label = required_text(prediction_row, "label", f"prediction row {index}") reference = reference_text(reference_row, index) if "row_index" in meta and meta["row_index"] != index: raise ValueError(f"metadata row {index}: row_index does not match the reference order") source_metadata = meta.get("source_metadata", {}) if not isinstance(source_metadata, dict): raise ValueError(f"metadata row {index}: source_metadata must be an object") task_type = meta.get( "task_type", source_metadata.get("task_type", reference_row.get("task_type", "unknown")) ) if not isinstance(task_type, str): raise ValueError(f"metadata row {index}: task_type must be a string") row = { "id": meta.get("stable_id", meta.get("id", reference_row.get("id", index))), "index": index, "task_type": task_type, "prediction": prediction, "reference": reference, "exported_label": label, "label_matches_full_reference": normalized_tokens(label) == normalized_tokens(reference), "label_matches_full_reference_raw": label == reference, **score_pair(prediction, reference), } rows.append(row) groups[task_type].append(row) metrics = { "schema_version": 1, "metric_scale": "0..100", "metric_version": "aya_overlap_v1", "overall": summarize(rows), "by_task_type": {name: summarize(group) for name, group in sorted(groups.items())}, "inputs": { "predictions": {"path": str(predictions_path.resolve()), "sha256": sha256(predictions_path)}, "references": {"path": str(references_path.resolve()), "sha256": sha256(references_path)}, }, } if metadata_path is not None: metrics["inputs"]["metadata"] = {"path": str(metadata_path.resolve()), "sha256": sha256(metadata_path)} if output_dir.exists() and (not output_dir.is_dir() or any(output_dir.iterdir())): raise ValueError(f"Output directory must be new or empty: {output_dir}") output_dir.mkdir(parents=True, exist_ok=True) with (output_dir / "per_sample.jsonl").open("w", encoding="utf-8", newline="\n") as handle: for row in rows: handle.write(json.dumps(row, ensure_ascii=False, allow_nan=False) + "\n") (output_dir / "metrics.json").write_text( json.dumps(metrics, ensure_ascii=False, indent=2, allow_nan=False) + "\n", encoding="utf-8" ) return metrics def evaluation_configs(training: dict, adapter: Path, output_dir: Path, split: str, max_new_tokens: int): keys = ("model_name_or_path", "model_revision", "stage", "finetuning_type", "template", "dataset_dir", "cutoff_len", "packing", "train_on_prompt", "bf16", "infer_dtype", "flash_attn", "use_unsloth", "rope_scaling", "seed", "data_seed", "preprocessing_num_workers", "dataloader_num_workers", "report_to") base = {key: training[key] for key in keys if key in training} base.update({"adapter_name_or_path": str(adapter), "eval_dataset": f"aya_{split}", "use_unsloth_gc": False, "overwrite_output_dir": False}) loss = dict(base, output_dir=str(output_dir / "loss"), do_eval=True, predict_with_generate=False, per_device_eval_batch_size=8) generation = dict(base, output_dir=str(output_dir / "generation"), do_predict=True, predict_with_generate=True, per_device_eval_batch_size=1, do_sample=False, num_beams=1, max_new_tokens=max_new_tokens) return loss, generation def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("config", type=Path, help="The same model YAML used for training.") parser.add_argument("--adapter", type=Path, help="Defaults to output_dir in the model YAML.") parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--split", choices=("eval", "test"), default="eval") parser.add_argument("--max-new-tokens", type=int, default=256) parser.add_argument("--dry-run", action="store_true") args = parser.parse_args() if args.max_new_tokens < 1: parser.error("--max-new-tokens must be positive.") config = read_config(args.config.resolve()) data_dir = Path(config["dataset_dir"]) manifest = verify_data(data_dir, (args.split,)) verify_length_filter(manifest, config) adapter = args.adapter.resolve() if args.adapter else Path(config["output_dir"]) if not (adapter / "adapter_config.json").is_file(): raise ValueError(f"Missing trained adapter: {adapter}") run_dir = verify_training_dataset(adapter, data_dir) used_config = run_dir / "used_config.yaml" if not used_config.is_file(): raise ValueError(f"Missing recorded training configuration: {used_config}") if used_config.is_file(): trained = read_config(used_config) for key in ("model_name_or_path", "model_revision", "template", "cutoff_len", "rope_scaling"): if trained.get(key) != config.get(key): raise ValueError(f"Evaluation setting {key} differs from this adapter's training; use {used_config}.") output_dir = args.output_dir.resolve() if output_dir.exists() and any(output_dir.iterdir()): raise ValueError(f"Evaluation output directory must be new or empty: {output_dir}") loss, generation = evaluation_configs(config, adapter, output_dir, args.split, args.max_new_tokens) if args.dry_run: print(json.dumps({"split": args.split, "adapter": str(adapter), "evaluation_gpus": 1, "loss_config": loss, "generation_config": generation}, ensure_ascii=False, indent=2)) print("Evaluation configuration/data checks passed. No inference started.") return 0 output_dir.mkdir(parents=True, exist_ok=True) record_path = output_dir / "evaluation.json" record = {"status": "running", "split": args.split, "adapter": str(adapter), "config": str(args.config.resolve()), "gpus": 1, "backend": "Unsloth", "precision": "bf16"} record_path.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8") env = os.environ.copy() env.update({"PYTHONUNBUFFERED": "1", "TOKENIZERS_PARALLELISM": "false"}) try: for name, settings in (("eval_loss", loss), ("eval_generate", generation)): path = output_dir / f"{name}.yaml" write_config(path, settings) command = [sys.executable, "-m", "torch.distributed.run", "--standalone", "--nproc_per_node=1", str(ROOT / "scripts/train.py"), str(path), "--worker"] run_logged(command, output_dir / f"{name}.log", env) metrics = score_files(output_dir / "generation/generated_predictions.jsonl", data_dir / f"{args.split}.jsonl", output_dir / "scores", data_dir / f"{args.split}.metadata.jsonl") reported = json.loads((output_dir / "loss/eval_results.json").read_text(encoding="utf-8")) value = reported.get("eval_loss") if value is None or not math.isfinite(value): raise ValueError("LLaMA-Factory did not report a finite eval_loss.") summary = {"model": config["model_name_or_path"], "split": args.split, "eval_loss": value, "perplexity": math.exp(value) if value < 700 else None, "token_f1": metrics["overall"]["token_f1"], "normalized_exact_match": metrics["overall"]["normalized_exact_match"], "examples": metrics["overall"]["count_total"], "metric_scale": "0..100", "max_new_tokens": args.max_new_tokens, "cutoff_len": config["cutoff_len"], "generation_context_limit": config["cutoff_len"]} (output_dir / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") record["status"] = "complete" print(json.dumps(summary, ensure_ascii=False, indent=2)) except BaseException: record["status"] = "failed_or_interrupted" raise finally: record_path.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8") return 0 if __name__ == "__main__": try: raise SystemExit(main()) except (OSError, ValueError, RuntimeError, ImportError, subprocess.CalledProcessError) as error: print(f"ERROR: {error}", file=sys.stderr) raise SystemExit(1)