"""Curate the completed 300k learning-rate SK-bench run without copying raw data. Requires Python 3.10+ and pyarrow (installed separately). The only read of the large details parquets projects model_response.truncated_tokens_count; row counts come from Parquet metadata. Cached GENERATIVE.parquet files are never read. """ from __future__ import annotations import argparse import collections import csv import hashlib import json import pathlib import re import shutil import statistics import sys from urllib.parse import unquote def write_csv(path, rows): if not rows: return columns = list(dict.fromkeys(k for row in rows for k in row)) with path.open('w', newline='', encoding='utf-8') as stream: writer = csv.DictWriter(stream, fieldnames=columns) writer.writeheader() writer.writerows(rows) def normalize(name): name = unquote(name.strip()) return re.sub(r'\|([0-9]+)$', r':\1', name) def get_lines(path): return [line.strip() for line in path.read_text(encoding='utf-8').splitlines() if line.strip() and not line.startswith('#')] def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--source', type=pathlib.Path, required=True, help='Extracted archive root containing skbench_evaluation/') parser.add_argument('--output', type=pathlib.Path, required=True) parser.add_argument('--pyarrow-path', type=pathlib.Path, help='Optional project-local pip --target directory') args = parser.parse_args() if args.pyarrow_path: sys.path.insert(0, str(args.pyarrow_path.resolve())) import pyarrow import pyarrow.parquet as pq source = args.source.resolve() out = args.output.resolve() base = source / 'skbench_evaluation' run_name = 'sk_zero_available_without_nli_lr_0p00010_20261022_185255' run = base / 'results' / run_name for name in ['results', 'logs', 'configs', 'provenance']: (out/name).mkdir(parents=True, exist_ok=True) result_path = run/'results/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010/results_2026-10-22T21-55-04.268099.json' d = json.loads(result_path.read_text(encoding='utf-8')) tasks = get_lines(run/'tasks.txt') omitted = get_lines(run/'omitted_tasks.txt') configs = d['config_tasks'] normalized_expected = {normalize(x) for x in tasks} assert len(tasks) == len(normalized_expected) == 222 assert normalized_expected == {normalize(x) for x in configs}, 'Config task coverage mismatch' assert normalized_expected <= {normalize(x) for x in d['results']}, 'Result task coverage mismatch' assert len(omitted) == 190 all_tasks = get_lines(base/'configs/tasks_sk_zero.txt') assert normalized_expected.isdisjoint({normalize(x) for x in omitted}) assert normalized_expected | {normalize(x) for x in omitted} == {normalize(x) for x in all_tasks} copies = {result_path: out/'results'/result_path.name, base/'logs'/f'{run_name}.log': out/'logs'/f'{run_name}.log', run/'tasks.txt': out/'configs/tasks.txt', run/'omitted_tasks.txt': out/'configs/omitted_tasks.txt', run/'uv.lock': out/'provenance/uv.lock', run/'skbench_commit.txt': out/'provenance/skbench_commit.txt', source/'archive_summary.json': out/'provenance/archive_summary.json'} for path in sorted((base/'configs').iterdir()): if path.is_file(): copies[path] = out/'configs'/path.name provenance = [] for src, dst in copies.items(): shutil.copyfile(src, dst) data = src.read_bytes() provenance.append({'source_archive_member': src.relative_to(source).as_posix(), 'curated_path': dst.relative_to(out).as_posix(), 'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest()}) write_csv(out/'provenance/copied_files_sha256.csv', provenance) (out/'configs/model_config.json').write_text(json.dumps(d['config_general'], indent=2, ensure_ascii=False)+'\n', encoding='utf-8') (out/'configs/task_configs.json').write_text(json.dumps(configs, indent=2, ensure_ascii=False)+'\n', encoding='utf-8') # Resolve the original names from the extraction manifest; Windows paths # encode ':' and '|' as %3A and %7C, while payload bytes were unchanged. manifest = json.loads((source/'archive_manifest.json').read_text(encoding='utf-8')) if isinstance(manifest, dict): manifest = manifest.get('entries', manifest.get('files', [])) original_paths = {x['local']: x['original'] for x in manifest if isinstance(x, dict) and 'local' in x} details = [] lookup = {} files = sorted((run/'details').rglob('*.parquet')) assert len(files) == 222, f'Expected 222 DETAILS files, got {len(files)}' for file in files: decoded = unquote(file.name) task = re.match(r'^details_(.+)_2026-10-22T21-55-04\.268099\.parquet$', decoded).group(1) pf = pq.ParquetFile(file) projected = pf.read(columns=['model_response.truncated_tokens_count']) values = projected.column('model_response').combine_chunks().field('truncated_tokens_count').to_pylist() assert len(values) == pf.metadata.num_rows assert all(v is None or (isinstance(v, int) and v >= 0) for v in values) row = {'task': task, 'sample_count': pf.metadata.num_rows, 'truncation_values_present': sum(v is not None for v in values), 'truncation_null_values': sum(v is None for v in values), 'samples_with_truncation': sum(v is not None and v > 0 for v in values), 'truncated_tokens_total': sum(v or 0 for v in values), 'max_truncated_tokens_per_sample': max((v or 0 for v in values), default=0), 'parquet_bytes': file.stat().st_size, 'source_archive_member': original_paths.get(file.relative_to(source).as_posix(), file.relative_to(source).as_posix()), 'extracted_relative_path': file.relative_to(source).as_posix()} assert normalize(task) not in lookup lookup[normalize(task)] = row details.append(row) assert set(lookup) == normalized_expected, 'DETAILS task coverage mismatch' write_csv(out/'results/truncation_audit_per_task.csv', details) metric_names = sorted({key for task in configs for key in d['results'][task]}) per_task = [] for task in sorted(configs): metric = d['results'][task] config = configs[task] detail = lookup[normalize(task)] primary = next((name for name in ['em_relaxed', 'em_norm_relaxed', 'exact_match_slk_prefix'] if name in metric), None) family = task.split('.prompt')[0] prompt = int(re.search(r'\.prompt([123])_', task).group(1)) task_group = re.sub(r'\.prompt[123]_', '.', task).removesuffix('|0') row = {'task': task, 'family': family, 'task_group_without_prompt': task_group, 'prompt': prompt, 'num_fewshots': config['num_fewshots'], 'sample_count': detail['sample_count'], 'primary_metric': primary, 'primary_value': metric[primary], 'generation_size': config['generation_size'], 'evaluation_splits': '|'.join(config['evaluation_splits']), 'hf_repo': config['hf_repo'], 'hf_subset': config['hf_subset'], 'hf_revision': config['hf_revision']} row.update({key: metric.get(key, '') for key in metric_names}) per_task.append(row) write_csv(out/'results/per_task_metrics.csv', per_task) # Each summary groups only identical metrics; values remain 0..1. # Macro means weight task variants equally. Sample-weighted means weight # their actual DETAILS rows. Repeated prompts are evaluation instances, # never additional independent underlying test questions. def summarize(group_columns, filename): groups = collections.defaultdict(list) for row in per_task: for metric_name in metric_names: if not metric_name.endswith('_stderr') and row[metric_name] != '': groups[tuple(row[x] for x in group_columns)+(metric_name,)].append(row) result = [] for key, rows in sorted(groups.items()): metric_name = key[-1] total = sum(x['sample_count'] for x in rows) item = dict(zip(group_columns, key[:-1])) item.update({'metric': metric_name, 'task_variants': len(rows), 'prompt_count': len({x['prompt'] for x in rows}), 'evaluated_instances': total, 'task_macro_mean': statistics.mean(x[metric_name] for x in rows), 'sample_weighted_mean': sum(x['sample_count']*x[metric_name] for x in rows)/total, 'min_task_value': min(x[metric_name] for x in rows), 'max_task_value': max(x[metric_name] for x in rows)}) result.append(item) write_csv(out/'results'/filename, result) return result family_summary = summarize(['family'], 'family_summary.csv') summarize(['family', 'prompt'], 'family_prompt_summary.csv') three_prompt = summarize(['family', 'task_group_without_prompt'], 'three_prompt_summary.csv') assert all(x['task_variants'] == x['prompt_count'] == 3 for x in three_prompt) access = json.loads((base/'configs/dataset_access_report.json').read_text(encoding='utf-8')) inaccessible = [x for x in access if x['status'] != 'OK'] with (base/'logs'/f'{run_name}.log').open(encoding='utf-8', newline='') as stream: log_text = stream.read() truncation_warnings = re.findall(r'context_size \+ max_new_tokens=\d+[^\r\n]*Truncating context to \d+ tokens\.', log_text) warning_evidence = [] for line_number, line in enumerate(log_text.split('\n'), 1): if 'Truncating context' in line: warning_evidence.append({'line_number': line_number, 'message': line[line.index('context_size'):], 'source': f'logs/{run_name}.log'}) (out/'results/log_truncation_evidence.json').write_text(json.dumps(warning_evidence, indent=2, ensure_ascii=False)+'\n', encoding='utf-8') summary = {'run_name': run_name, 'result_file': result_path.name, 'task_list_count': len(tasks), 'configured_tasks': len(configs), 'scored_task_variants': len(per_task), 'result_json_keys': len(d['results']), 'extra_result_keys_excluded_from_task_summaries': sorted(set(d['results'])-set(configs)), 'all_original_task_variants': len(all_tasks), 'omitted_task_variants': len(omitted), 'inaccessible_repositories': len(inaccessible), 'inaccessible_task_variants': sum(len(x['tasks']) for x in inaccessible), 'additionally_excluded_nli_task_variants': 3, 'families': dict(sorted(collections.Counter(row['family'] for row in per_task).items())), 'prompt_counts': dict(sorted(collections.Counter(row['prompt'] for row in per_task).items())), 'evaluated_instances': sum(row['sample_count'] for row in per_task), 'instances_by_prompt': {str(prompt): sum(x['sample_count'] for x in per_task if x['prompt'] == prompt) for prompt in [1,2,3]}, 'details_files_audited': len(details), 'truncation_column': 'model_response.truncated_tokens_count', 'truncation_values_present': sum(x['truncation_values_present'] for x in details), 'truncation_null_values': sum(x['truncation_null_values'] for x in details), 'details_reported_samples_with_truncation': sum(x['samples_with_truncation'] for x in details), 'details_reported_truncated_tokens_total': sum(x['truncated_tokens_total'] for x in details), 'details_reported_max_truncated_tokens_per_sample': max(x['max_truncated_tokens_per_sample'] for x in details), 'context_truncation_observed_in_log': bool(truncation_warnings), 'actual_truncated_sample_count': None, 'console_context_truncation_warning_count': len(truncation_warnings), 'console_context_truncation_warnings': truncation_warnings, 'truncation_interpretation': 'All DETAILS counters are zero, but the console explicitly reports context truncation. Do not infer no truncation or an affected-sample count from these counters.', 'model_config': d['config_general']['model_config'], 'max_samples': d['config_general']['max_samples'], 'generation_size_counts': dict(collections.Counter(str(x['generation_size']) for x in per_task)), 'original_summary_general': d['summary_general'], 'pyarrow_version': pyarrow.__version__, 'python_version': sys.version, 'aggregation': 'Descriptive metric-specific task macro means and sample-weighted means; no official full-SK-bench overall score.', 'limitations': ['Partial available zero-shot subset: 222 of 412 task variants.', 'Only the validation-selected LR=0.00010 was benchmarked.', 'Sample counts are evaluation instances; prompt variants reuse underlying questions.', 'One console warning reports context truncation; all DETAILS counters are zero, so the recorded counters do not reliably enumerate affected inputs.', 'The audit cannot recover the number of affected tasks/samples or quantify output generation-size termination.', 'All-zero summary counters include non_truncated=0 and cannot prove absence of truncation.', 'Raw DETAILS and generation-cache parquets are excluded from this curated tree.']} (out/'results/audit_summary.json').write_text(json.dumps(summary, indent=2, ensure_ascii=False)+'\n', encoding='utf-8') script_destination = out/'provenance/audit_skbench.py' if pathlib.Path(__file__).resolve() != script_destination.resolve(): shutil.copyfile(pathlib.Path(__file__), script_destination) print(json.dumps({key: summary[key] for key in ['scored_task_variants', 'evaluated_instances', 'instances_by_prompt', 'details_reported_samples_with_truncation', 'context_truncation_observed_in_log', 'actual_truncated_sample_count', 'truncation_null_values', 'families']}, indent=2)) print('Primary family macro means (metrics kept separate):') for row in family_summary: if row['metric'] in ['em_relaxed', 'em_norm_relaxed', 'exact_match_slk_prefix', 'f1_slk']: print(row) if __name__ == '__main__': main()