Diplomovka/aya_300k_lr_experiments/benchmarks/skbench/provenance/audit_skbench.py
Jakub 67061cf373 Add Aya 300k learning-rate study and benchmark results
Document six completed LR runs as preparation for full Aya training on Perun. Record selection of LR 0.00010 by lowest final validation loss, selected-model lm-eval and partial SK-bench results, reproducible provenance, and dataset-access/context-truncation limitations. Preserve existing 30k and Titan material.
2026-10-25 00:33:24 +02:00

235 lines
14 KiB
Python

"""Curate the completed 300k learning-rate SK-bench run without copying raw data.
Requires Python 3.10+ and pyarrow (installed separately). The only read of the
large details parquets projects model_response.truncated_tokens_count; row counts
come from Parquet metadata. Cached GENERATIVE.parquet files are never read.
"""
from __future__ import annotations
import argparse
import collections
import csv
import hashlib
import json
import pathlib
import re
import shutil
import statistics
import sys
from urllib.parse import unquote
def write_csv(path, rows):
if not rows:
return
columns = list(dict.fromkeys(k for row in rows for k in row))
with path.open('w', newline='', encoding='utf-8') as stream:
writer = csv.DictWriter(stream, fieldnames=columns)
writer.writeheader()
writer.writerows(rows)
def normalize(name):
name = unquote(name.strip())
return re.sub(r'\|([0-9]+)$', r':\1', name)
def get_lines(path):
return [line.strip() for line in path.read_text(encoding='utf-8').splitlines() if line.strip() and not line.startswith('#')]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--source', type=pathlib.Path, required=True, help='Extracted archive root containing skbench_evaluation/')
parser.add_argument('--output', type=pathlib.Path, required=True)
parser.add_argument('--pyarrow-path', type=pathlib.Path, help='Optional project-local pip --target directory')
args = parser.parse_args()
if args.pyarrow_path:
sys.path.insert(0, str(args.pyarrow_path.resolve()))
import pyarrow
import pyarrow.parquet as pq
source = args.source.resolve()
out = args.output.resolve()
base = source / 'skbench_evaluation'
run_name = 'sk_zero_available_without_nli_lr_0p00010_20261022_185255'
run = base / 'results' / run_name
for name in ['results', 'logs', 'configs', 'provenance']:
(out/name).mkdir(parents=True, exist_ok=True)
result_path = run/'results/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010/results_2026-10-22T21-55-04.268099.json'
d = json.loads(result_path.read_text(encoding='utf-8'))
tasks = get_lines(run/'tasks.txt')
omitted = get_lines(run/'omitted_tasks.txt')
configs = d['config_tasks']
normalized_expected = {normalize(x) for x in tasks}
assert len(tasks) == len(normalized_expected) == 222
assert normalized_expected == {normalize(x) for x in configs}, 'Config task coverage mismatch'
assert normalized_expected <= {normalize(x) for x in d['results']}, 'Result task coverage mismatch'
assert len(omitted) == 190
all_tasks = get_lines(base/'configs/tasks_sk_zero.txt')
assert normalized_expected.isdisjoint({normalize(x) for x in omitted})
assert normalized_expected | {normalize(x) for x in omitted} == {normalize(x) for x in all_tasks}
copies = {result_path: out/'results'/result_path.name,
base/'logs'/f'{run_name}.log': out/'logs'/f'{run_name}.log',
run/'tasks.txt': out/'configs/tasks.txt',
run/'omitted_tasks.txt': out/'configs/omitted_tasks.txt',
run/'uv.lock': out/'provenance/uv.lock',
run/'skbench_commit.txt': out/'provenance/skbench_commit.txt',
source/'archive_summary.json': out/'provenance/archive_summary.json'}
for path in sorted((base/'configs').iterdir()):
if path.is_file():
copies[path] = out/'configs'/path.name
provenance = []
for src, dst in copies.items():
shutil.copyfile(src, dst)
data = src.read_bytes()
provenance.append({'source_archive_member': src.relative_to(source).as_posix(),
'curated_path': dst.relative_to(out).as_posix(),
'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest()})
write_csv(out/'provenance/copied_files_sha256.csv', provenance)
(out/'configs/model_config.json').write_text(json.dumps(d['config_general'], indent=2, ensure_ascii=False)+'\n', encoding='utf-8')
(out/'configs/task_configs.json').write_text(json.dumps(configs, indent=2, ensure_ascii=False)+'\n', encoding='utf-8')
# Resolve the original names from the extraction manifest; Windows paths
# encode ':' and '|' as %3A and %7C, while payload bytes were unchanged.
manifest = json.loads((source/'archive_manifest.json').read_text(encoding='utf-8'))
if isinstance(manifest, dict):
manifest = manifest.get('entries', manifest.get('files', []))
original_paths = {x['local']: x['original'] for x in manifest if isinstance(x, dict) and 'local' in x}
details = []
lookup = {}
files = sorted((run/'details').rglob('*.parquet'))
assert len(files) == 222, f'Expected 222 DETAILS files, got {len(files)}'
for file in files:
decoded = unquote(file.name)
task = re.match(r'^details_(.+)_2026-10-22T21-55-04\.268099\.parquet$', decoded).group(1)
pf = pq.ParquetFile(file)
projected = pf.read(columns=['model_response.truncated_tokens_count'])
values = projected.column('model_response').combine_chunks().field('truncated_tokens_count').to_pylist()
assert len(values) == pf.metadata.num_rows
assert all(v is None or (isinstance(v, int) and v >= 0) for v in values)
row = {'task': task, 'sample_count': pf.metadata.num_rows,
'truncation_values_present': sum(v is not None for v in values),
'truncation_null_values': sum(v is None for v in values),
'samples_with_truncation': sum(v is not None and v > 0 for v in values),
'truncated_tokens_total': sum(v or 0 for v in values),
'max_truncated_tokens_per_sample': max((v or 0 for v in values), default=0),
'parquet_bytes': file.stat().st_size,
'source_archive_member': original_paths.get(file.relative_to(source).as_posix(), file.relative_to(source).as_posix()),
'extracted_relative_path': file.relative_to(source).as_posix()}
assert normalize(task) not in lookup
lookup[normalize(task)] = row
details.append(row)
assert set(lookup) == normalized_expected, 'DETAILS task coverage mismatch'
write_csv(out/'results/truncation_audit_per_task.csv', details)
metric_names = sorted({key for task in configs for key in d['results'][task]})
per_task = []
for task in sorted(configs):
metric = d['results'][task]
config = configs[task]
detail = lookup[normalize(task)]
primary = next((name for name in ['em_relaxed', 'em_norm_relaxed', 'exact_match_slk_prefix'] if name in metric), None)
family = task.split('.prompt')[0]
prompt = int(re.search(r'\.prompt([123])_', task).group(1))
task_group = re.sub(r'\.prompt[123]_', '.', task).removesuffix('|0')
row = {'task': task, 'family': family, 'task_group_without_prompt': task_group,
'prompt': prompt, 'num_fewshots': config['num_fewshots'], 'sample_count': detail['sample_count'],
'primary_metric': primary, 'primary_value': metric[primary],
'generation_size': config['generation_size'], 'evaluation_splits': '|'.join(config['evaluation_splits']),
'hf_repo': config['hf_repo'], 'hf_subset': config['hf_subset'], 'hf_revision': config['hf_revision']}
row.update({key: metric.get(key, '') for key in metric_names})
per_task.append(row)
write_csv(out/'results/per_task_metrics.csv', per_task)
# Each summary groups only identical metrics; values remain 0..1.
# Macro means weight task variants equally. Sample-weighted means weight
# their actual DETAILS rows. Repeated prompts are evaluation instances,
# never additional independent underlying test questions.
def summarize(group_columns, filename):
groups = collections.defaultdict(list)
for row in per_task:
for metric_name in metric_names:
if not metric_name.endswith('_stderr') and row[metric_name] != '':
groups[tuple(row[x] for x in group_columns)+(metric_name,)].append(row)
result = []
for key, rows in sorted(groups.items()):
metric_name = key[-1]
total = sum(x['sample_count'] for x in rows)
item = dict(zip(group_columns, key[:-1]))
item.update({'metric': metric_name, 'task_variants': len(rows),
'prompt_count': len({x['prompt'] for x in rows}),
'evaluated_instances': total,
'task_macro_mean': statistics.mean(x[metric_name] for x in rows),
'sample_weighted_mean': sum(x['sample_count']*x[metric_name] for x in rows)/total,
'min_task_value': min(x[metric_name] for x in rows),
'max_task_value': max(x[metric_name] for x in rows)})
result.append(item)
write_csv(out/'results'/filename, result)
return result
family_summary = summarize(['family'], 'family_summary.csv')
summarize(['family', 'prompt'], 'family_prompt_summary.csv')
three_prompt = summarize(['family', 'task_group_without_prompt'], 'three_prompt_summary.csv')
assert all(x['task_variants'] == x['prompt_count'] == 3 for x in three_prompt)
access = json.loads((base/'configs/dataset_access_report.json').read_text(encoding='utf-8'))
inaccessible = [x for x in access if x['status'] != 'OK']
with (base/'logs'/f'{run_name}.log').open(encoding='utf-8', newline='') as stream:
log_text = stream.read()
truncation_warnings = re.findall(r'context_size \+ max_new_tokens=\d+[^\r\n]*Truncating context to \d+ tokens\.', log_text)
warning_evidence = []
for line_number, line in enumerate(log_text.split('\n'), 1):
if 'Truncating context' in line:
warning_evidence.append({'line_number': line_number, 'message': line[line.index('context_size'):], 'source': f'logs/{run_name}.log'})
(out/'results/log_truncation_evidence.json').write_text(json.dumps(warning_evidence, indent=2, ensure_ascii=False)+'\n', encoding='utf-8')
summary = {'run_name': run_name, 'result_file': result_path.name,
'task_list_count': len(tasks), 'configured_tasks': len(configs), 'scored_task_variants': len(per_task),
'result_json_keys': len(d['results']),
'extra_result_keys_excluded_from_task_summaries': sorted(set(d['results'])-set(configs)),
'all_original_task_variants': len(all_tasks),
'omitted_task_variants': len(omitted), 'inaccessible_repositories': len(inaccessible),
'inaccessible_task_variants': sum(len(x['tasks']) for x in inaccessible),
'additionally_excluded_nli_task_variants': 3,
'families': dict(sorted(collections.Counter(row['family'] for row in per_task).items())),
'prompt_counts': dict(sorted(collections.Counter(row['prompt'] for row in per_task).items())),
'evaluated_instances': sum(row['sample_count'] for row in per_task),
'instances_by_prompt': {str(prompt): sum(x['sample_count'] for x in per_task if x['prompt'] == prompt) for prompt in [1,2,3]},
'details_files_audited': len(details), 'truncation_column': 'model_response.truncated_tokens_count',
'truncation_values_present': sum(x['truncation_values_present'] for x in details),
'truncation_null_values': sum(x['truncation_null_values'] for x in details),
'details_reported_samples_with_truncation': sum(x['samples_with_truncation'] for x in details),
'details_reported_truncated_tokens_total': sum(x['truncated_tokens_total'] for x in details),
'details_reported_max_truncated_tokens_per_sample': max(x['max_truncated_tokens_per_sample'] for x in details),
'context_truncation_observed_in_log': bool(truncation_warnings),
'actual_truncated_sample_count': None,
'console_context_truncation_warning_count': len(truncation_warnings),
'console_context_truncation_warnings': truncation_warnings,
'truncation_interpretation': 'All DETAILS counters are zero, but the console explicitly reports context truncation. Do not infer no truncation or an affected-sample count from these counters.',
'model_config': d['config_general']['model_config'],
'max_samples': d['config_general']['max_samples'],
'generation_size_counts': dict(collections.Counter(str(x['generation_size']) for x in per_task)),
'original_summary_general': d['summary_general'],
'pyarrow_version': pyarrow.__version__, 'python_version': sys.version,
'aggregation': 'Descriptive metric-specific task macro means and sample-weighted means; no official full-SK-bench overall score.',
'limitations': ['Partial available zero-shot subset: 222 of 412 task variants.',
'Only the validation-selected LR=0.00010 was benchmarked.',
'Sample counts are evaluation instances; prompt variants reuse underlying questions.',
'One console warning reports context truncation; all DETAILS counters are zero, so the recorded counters do not reliably enumerate affected inputs.',
'The audit cannot recover the number of affected tasks/samples or quantify output generation-size termination.',
'All-zero summary counters include non_truncated=0 and cannot prove absence of truncation.',
'Raw DETAILS and generation-cache parquets are excluded from this curated tree.']}
(out/'results/audit_summary.json').write_text(json.dumps(summary, indent=2, ensure_ascii=False)+'\n', encoding='utf-8')
script_destination = out/'provenance/audit_skbench.py'
if pathlib.Path(__file__).resolve() != script_destination.resolve():
shutil.copyfile(pathlib.Path(__file__), script_destination)
print(json.dumps({key: summary[key] for key in ['scored_task_variants', 'evaluated_instances', 'instances_by_prompt', 'details_reported_samples_with_truncation', 'context_truncation_observed_in_log', 'actual_truncated_sample_count', 'truncation_null_values', 'families']}, indent=2))
print('Primary family macro means (metrics kept separate):')
for row in family_summary:
if row['metric'] in ['em_relaxed', 'em_norm_relaxed', 'exact_match_slk_prefix', 'f1_slk']:
print(row)
if __name__ == '__main__':
main()