Document six completed LR runs as preparation for full Aya training on Perun. Record selection of LR 0.00010 by lowest final validation loss, selected-model lm-eval and partial SK-bench results, reproducible provenance, and dataset-access/context-truncation limitations. Preserve existing 30k and Titan material.
235 lines
14 KiB
Python
235 lines
14 KiB
Python
"""Curate the completed 300k learning-rate SK-bench run without copying raw data.
|
|
|
|
Requires Python 3.10+ and pyarrow (installed separately). The only read of the
|
|
large details parquets projects model_response.truncated_tokens_count; row counts
|
|
come from Parquet metadata. Cached GENERATIVE.parquet files are never read.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import collections
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import pathlib
|
|
import re
|
|
import shutil
|
|
import statistics
|
|
import sys
|
|
from urllib.parse import unquote
|
|
|
|
|
|
def write_csv(path, rows):
|
|
if not rows:
|
|
return
|
|
columns = list(dict.fromkeys(k for row in rows for k in row))
|
|
with path.open('w', newline='', encoding='utf-8') as stream:
|
|
writer = csv.DictWriter(stream, fieldnames=columns)
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
|
|
|
|
def normalize(name):
|
|
name = unquote(name.strip())
|
|
return re.sub(r'\|([0-9]+)$', r':\1', name)
|
|
|
|
|
|
def get_lines(path):
|
|
return [line.strip() for line in path.read_text(encoding='utf-8').splitlines() if line.strip() and not line.startswith('#')]
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--source', type=pathlib.Path, required=True, help='Extracted archive root containing skbench_evaluation/')
|
|
parser.add_argument('--output', type=pathlib.Path, required=True)
|
|
parser.add_argument('--pyarrow-path', type=pathlib.Path, help='Optional project-local pip --target directory')
|
|
args = parser.parse_args()
|
|
if args.pyarrow_path:
|
|
sys.path.insert(0, str(args.pyarrow_path.resolve()))
|
|
import pyarrow
|
|
import pyarrow.parquet as pq
|
|
|
|
source = args.source.resolve()
|
|
out = args.output.resolve()
|
|
base = source / 'skbench_evaluation'
|
|
run_name = 'sk_zero_available_without_nli_lr_0p00010_20261022_185255'
|
|
run = base / 'results' / run_name
|
|
for name in ['results', 'logs', 'configs', 'provenance']:
|
|
(out/name).mkdir(parents=True, exist_ok=True)
|
|
result_path = run/'results/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010/results_2026-10-22T21-55-04.268099.json'
|
|
d = json.loads(result_path.read_text(encoding='utf-8'))
|
|
tasks = get_lines(run/'tasks.txt')
|
|
omitted = get_lines(run/'omitted_tasks.txt')
|
|
configs = d['config_tasks']
|
|
normalized_expected = {normalize(x) for x in tasks}
|
|
assert len(tasks) == len(normalized_expected) == 222
|
|
assert normalized_expected == {normalize(x) for x in configs}, 'Config task coverage mismatch'
|
|
assert normalized_expected <= {normalize(x) for x in d['results']}, 'Result task coverage mismatch'
|
|
assert len(omitted) == 190
|
|
all_tasks = get_lines(base/'configs/tasks_sk_zero.txt')
|
|
assert normalized_expected.isdisjoint({normalize(x) for x in omitted})
|
|
assert normalized_expected | {normalize(x) for x in omitted} == {normalize(x) for x in all_tasks}
|
|
|
|
copies = {result_path: out/'results'/result_path.name,
|
|
base/'logs'/f'{run_name}.log': out/'logs'/f'{run_name}.log',
|
|
run/'tasks.txt': out/'configs/tasks.txt',
|
|
run/'omitted_tasks.txt': out/'configs/omitted_tasks.txt',
|
|
run/'uv.lock': out/'provenance/uv.lock',
|
|
run/'skbench_commit.txt': out/'provenance/skbench_commit.txt',
|
|
source/'archive_summary.json': out/'provenance/archive_summary.json'}
|
|
for path in sorted((base/'configs').iterdir()):
|
|
if path.is_file():
|
|
copies[path] = out/'configs'/path.name
|
|
provenance = []
|
|
for src, dst in copies.items():
|
|
shutil.copyfile(src, dst)
|
|
data = src.read_bytes()
|
|
provenance.append({'source_archive_member': src.relative_to(source).as_posix(),
|
|
'curated_path': dst.relative_to(out).as_posix(),
|
|
'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest()})
|
|
write_csv(out/'provenance/copied_files_sha256.csv', provenance)
|
|
(out/'configs/model_config.json').write_text(json.dumps(d['config_general'], indent=2, ensure_ascii=False)+'\n', encoding='utf-8')
|
|
(out/'configs/task_configs.json').write_text(json.dumps(configs, indent=2, ensure_ascii=False)+'\n', encoding='utf-8')
|
|
|
|
# Resolve the original names from the extraction manifest; Windows paths
|
|
# encode ':' and '|' as %3A and %7C, while payload bytes were unchanged.
|
|
manifest = json.loads((source/'archive_manifest.json').read_text(encoding='utf-8'))
|
|
if isinstance(manifest, dict):
|
|
manifest = manifest.get('entries', manifest.get('files', []))
|
|
original_paths = {x['local']: x['original'] for x in manifest if isinstance(x, dict) and 'local' in x}
|
|
details = []
|
|
lookup = {}
|
|
files = sorted((run/'details').rglob('*.parquet'))
|
|
assert len(files) == 222, f'Expected 222 DETAILS files, got {len(files)}'
|
|
for file in files:
|
|
decoded = unquote(file.name)
|
|
task = re.match(r'^details_(.+)_2026-10-22T21-55-04\.268099\.parquet$', decoded).group(1)
|
|
pf = pq.ParquetFile(file)
|
|
projected = pf.read(columns=['model_response.truncated_tokens_count'])
|
|
values = projected.column('model_response').combine_chunks().field('truncated_tokens_count').to_pylist()
|
|
assert len(values) == pf.metadata.num_rows
|
|
assert all(v is None or (isinstance(v, int) and v >= 0) for v in values)
|
|
row = {'task': task, 'sample_count': pf.metadata.num_rows,
|
|
'truncation_values_present': sum(v is not None for v in values),
|
|
'truncation_null_values': sum(v is None for v in values),
|
|
'samples_with_truncation': sum(v is not None and v > 0 for v in values),
|
|
'truncated_tokens_total': sum(v or 0 for v in values),
|
|
'max_truncated_tokens_per_sample': max((v or 0 for v in values), default=0),
|
|
'parquet_bytes': file.stat().st_size,
|
|
'source_archive_member': original_paths.get(file.relative_to(source).as_posix(), file.relative_to(source).as_posix()),
|
|
'extracted_relative_path': file.relative_to(source).as_posix()}
|
|
assert normalize(task) not in lookup
|
|
lookup[normalize(task)] = row
|
|
details.append(row)
|
|
assert set(lookup) == normalized_expected, 'DETAILS task coverage mismatch'
|
|
write_csv(out/'results/truncation_audit_per_task.csv', details)
|
|
|
|
metric_names = sorted({key for task in configs for key in d['results'][task]})
|
|
per_task = []
|
|
for task in sorted(configs):
|
|
metric = d['results'][task]
|
|
config = configs[task]
|
|
detail = lookup[normalize(task)]
|
|
primary = next((name for name in ['em_relaxed', 'em_norm_relaxed', 'exact_match_slk_prefix'] if name in metric), None)
|
|
family = task.split('.prompt')[0]
|
|
prompt = int(re.search(r'\.prompt([123])_', task).group(1))
|
|
task_group = re.sub(r'\.prompt[123]_', '.', task).removesuffix('|0')
|
|
row = {'task': task, 'family': family, 'task_group_without_prompt': task_group,
|
|
'prompt': prompt, 'num_fewshots': config['num_fewshots'], 'sample_count': detail['sample_count'],
|
|
'primary_metric': primary, 'primary_value': metric[primary],
|
|
'generation_size': config['generation_size'], 'evaluation_splits': '|'.join(config['evaluation_splits']),
|
|
'hf_repo': config['hf_repo'], 'hf_subset': config['hf_subset'], 'hf_revision': config['hf_revision']}
|
|
row.update({key: metric.get(key, '') for key in metric_names})
|
|
per_task.append(row)
|
|
write_csv(out/'results/per_task_metrics.csv', per_task)
|
|
|
|
# Each summary groups only identical metrics; values remain 0..1.
|
|
# Macro means weight task variants equally. Sample-weighted means weight
|
|
# their actual DETAILS rows. Repeated prompts are evaluation instances,
|
|
# never additional independent underlying test questions.
|
|
def summarize(group_columns, filename):
|
|
groups = collections.defaultdict(list)
|
|
for row in per_task:
|
|
for metric_name in metric_names:
|
|
if not metric_name.endswith('_stderr') and row[metric_name] != '':
|
|
groups[tuple(row[x] for x in group_columns)+(metric_name,)].append(row)
|
|
result = []
|
|
for key, rows in sorted(groups.items()):
|
|
metric_name = key[-1]
|
|
total = sum(x['sample_count'] for x in rows)
|
|
item = dict(zip(group_columns, key[:-1]))
|
|
item.update({'metric': metric_name, 'task_variants': len(rows),
|
|
'prompt_count': len({x['prompt'] for x in rows}),
|
|
'evaluated_instances': total,
|
|
'task_macro_mean': statistics.mean(x[metric_name] for x in rows),
|
|
'sample_weighted_mean': sum(x['sample_count']*x[metric_name] for x in rows)/total,
|
|
'min_task_value': min(x[metric_name] for x in rows),
|
|
'max_task_value': max(x[metric_name] for x in rows)})
|
|
result.append(item)
|
|
write_csv(out/'results'/filename, result)
|
|
return result
|
|
family_summary = summarize(['family'], 'family_summary.csv')
|
|
summarize(['family', 'prompt'], 'family_prompt_summary.csv')
|
|
three_prompt = summarize(['family', 'task_group_without_prompt'], 'three_prompt_summary.csv')
|
|
assert all(x['task_variants'] == x['prompt_count'] == 3 for x in three_prompt)
|
|
|
|
access = json.loads((base/'configs/dataset_access_report.json').read_text(encoding='utf-8'))
|
|
inaccessible = [x for x in access if x['status'] != 'OK']
|
|
with (base/'logs'/f'{run_name}.log').open(encoding='utf-8', newline='') as stream:
|
|
log_text = stream.read()
|
|
truncation_warnings = re.findall(r'context_size \+ max_new_tokens=\d+[^\r\n]*Truncating context to \d+ tokens\.', log_text)
|
|
warning_evidence = []
|
|
for line_number, line in enumerate(log_text.split('\n'), 1):
|
|
if 'Truncating context' in line:
|
|
warning_evidence.append({'line_number': line_number, 'message': line[line.index('context_size'):], 'source': f'logs/{run_name}.log'})
|
|
(out/'results/log_truncation_evidence.json').write_text(json.dumps(warning_evidence, indent=2, ensure_ascii=False)+'\n', encoding='utf-8')
|
|
summary = {'run_name': run_name, 'result_file': result_path.name,
|
|
'task_list_count': len(tasks), 'configured_tasks': len(configs), 'scored_task_variants': len(per_task),
|
|
'result_json_keys': len(d['results']),
|
|
'extra_result_keys_excluded_from_task_summaries': sorted(set(d['results'])-set(configs)),
|
|
'all_original_task_variants': len(all_tasks),
|
|
'omitted_task_variants': len(omitted), 'inaccessible_repositories': len(inaccessible),
|
|
'inaccessible_task_variants': sum(len(x['tasks']) for x in inaccessible),
|
|
'additionally_excluded_nli_task_variants': 3,
|
|
'families': dict(sorted(collections.Counter(row['family'] for row in per_task).items())),
|
|
'prompt_counts': dict(sorted(collections.Counter(row['prompt'] for row in per_task).items())),
|
|
'evaluated_instances': sum(row['sample_count'] for row in per_task),
|
|
'instances_by_prompt': {str(prompt): sum(x['sample_count'] for x in per_task if x['prompt'] == prompt) for prompt in [1,2,3]},
|
|
'details_files_audited': len(details), 'truncation_column': 'model_response.truncated_tokens_count',
|
|
'truncation_values_present': sum(x['truncation_values_present'] for x in details),
|
|
'truncation_null_values': sum(x['truncation_null_values'] for x in details),
|
|
'details_reported_samples_with_truncation': sum(x['samples_with_truncation'] for x in details),
|
|
'details_reported_truncated_tokens_total': sum(x['truncated_tokens_total'] for x in details),
|
|
'details_reported_max_truncated_tokens_per_sample': max(x['max_truncated_tokens_per_sample'] for x in details),
|
|
'context_truncation_observed_in_log': bool(truncation_warnings),
|
|
'actual_truncated_sample_count': None,
|
|
'console_context_truncation_warning_count': len(truncation_warnings),
|
|
'console_context_truncation_warnings': truncation_warnings,
|
|
'truncation_interpretation': 'All DETAILS counters are zero, but the console explicitly reports context truncation. Do not infer no truncation or an affected-sample count from these counters.',
|
|
'model_config': d['config_general']['model_config'],
|
|
'max_samples': d['config_general']['max_samples'],
|
|
'generation_size_counts': dict(collections.Counter(str(x['generation_size']) for x in per_task)),
|
|
'original_summary_general': d['summary_general'],
|
|
'pyarrow_version': pyarrow.__version__, 'python_version': sys.version,
|
|
'aggregation': 'Descriptive metric-specific task macro means and sample-weighted means; no official full-SK-bench overall score.',
|
|
'limitations': ['Partial available zero-shot subset: 222 of 412 task variants.',
|
|
'Only the validation-selected LR=0.00010 was benchmarked.',
|
|
'Sample counts are evaluation instances; prompt variants reuse underlying questions.',
|
|
'One console warning reports context truncation; all DETAILS counters are zero, so the recorded counters do not reliably enumerate affected inputs.',
|
|
'The audit cannot recover the number of affected tasks/samples or quantify output generation-size termination.',
|
|
'All-zero summary counters include non_truncated=0 and cannot prove absence of truncation.',
|
|
'Raw DETAILS and generation-cache parquets are excluded from this curated tree.']}
|
|
(out/'results/audit_summary.json').write_text(json.dumps(summary, indent=2, ensure_ascii=False)+'\n', encoding='utf-8')
|
|
script_destination = out/'provenance/audit_skbench.py'
|
|
if pathlib.Path(__file__).resolve() != script_destination.resolve():
|
|
shutil.copyfile(pathlib.Path(__file__), script_destination)
|
|
print(json.dumps({key: summary[key] for key in ['scored_task_variants', 'evaluated_instances', 'instances_by_prompt', 'details_reported_samples_with_truncation', 'context_truncation_observed_in_log', 'actual_truncated_sample_count', 'truncation_null_values', 'families']}, indent=2))
|
|
print('Primary family macro means (metrics kept separate):')
|
|
for row in family_summary:
|
|
if row['metric'] in ['em_relaxed', 'em_norm_relaxed', 'exact_match_slk_prefix', 'f1_slk']:
|
|
print(row)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|