{
  "run_name": "sk_zero_available_without_nli_lr_0p00010_20261022_185255",
  "result_file": "results_2026-10-22T21-55-04.268099.json",
  "task_list_count": 222,
  "configured_tasks": 222,
  "scored_task_variants": 222,
  "result_json_keys": 226,
  "extra_result_keys_excluded_from_task_summaries": [
    "all",
    "mmlu.prompt1_mqa_single_correct_sk:_average|0",
    "mmlu.prompt2_mqa_single_correct_sk:_average|0",
    "mmlu.prompt3_mqa_single_correct_sk:_average|0"
  ],
  "all_original_task_variants": 412,
  "omitted_task_variants": 190,
  "inaccessible_repositories": 12,
  "inaccessible_task_variants": 187,
  "additionally_excluded_nli_task_variants": 3,
  "families": {
    "arc": 3,
    "belebele": 3,
    "financial_exams": 3,
    "globalmmlu-lite": 3,
    "mmlu": 171,
    "movie_review": 3,
    "multi_wiki_qa": 3,
    "senti_sk": 6,
    "skj1": 3,
    "sklep": 15,
    "slovakcopa": 3,
    "toxic_sk": 3,
    "truthfulqa": 3
  },
  "prompt_counts": {
    "1": 74,
    "2": 74,
    "3": 74
  },
  "evaluated_instances": 159426,
  "instances_by_prompt": {
    "1": 53142,
    "2": 53142,
    "3": 53142
  },
  "details_files_audited": 222,
  "truncation_column": "model_response.truncated_tokens_count",
  "truncation_values_present": 159426,
  "truncation_null_values": 0,
  "details_reported_samples_with_truncation": 0,
  "details_reported_truncated_tokens_total": 0,
  "details_reported_max_truncated_tokens_per_sample": 0,
  "context_truncation_observed_in_log": true,
  "actual_truncated_sample_count": null,
  "console_context_truncation_warning_count": 1,
  "console_context_truncation_warnings": [
    "context_size + max_new_tokens=53860 which is greater than self.max_length=32768. Truncating context to 32642 tokens."
  ],
  "truncation_interpretation": "All DETAILS counters are zero, but the console explicitly reports context truncation. Do not infer no truncation or an affected-sample count from these counters.",
  "model_config": {
    "model_name": "/home/schwarc/diplomovka/skbench_evaluation/models/aya_300k_lr_0p00010",
    "generation_parameters": {
      "num_blocks": null,
      "block_size": null,
      "early_stopping": null,
      "repetition_penalty": null,
      "frequency_penalty": null,
      "length_penalty": null,
      "presence_penalty": null,
      "max_new_tokens": null,
      "min_new_tokens": null,
      "seed": null,
      "stop_tokens": null,
      "temperature": 0,
      "top_k": null,
      "min_p": null,
      "top_p": null,
      "truncate_prompt": null,
      "cache_implementation": null,
      "response_format": null,
      "skip_special_tokens": true
    },
    "system_prompt": null,
    "cache_dir": "~/.cache/huggingface/lighteval",
    "tokenizer": null,
    "revision": "main",
    "dtype": "bfloat16",
    "tensor_parallel_size": 1,
    "data_parallel_size": 1,
    "pipeline_parallel_size": 1,
    "gpu_memory_utilization": 0.85,
    "enable_prefix_caching": null,
    "max_model_length": 32768,
    "quantization": null,
    "load_format": null,
    "swap_space": 4,
    "seed": 1234,
    "trust_remote_code": false,
    "add_special_tokens": true,
    "multichoice_continuations_start_space": true,
    "pairwise_tokenization": false,
    "max_num_seqs": 128,
    "max_num_batched_tokens": 2048,
    "subfolder": null,
    "is_async": false,
    "override_chat_template": null,
    "enforce_eager": true
  },
  "max_samples": null,
  "generation_size_counts": {
    "32": 216,
    "126": 6
  },
  "original_summary_general": {
    "hashes": {
      "hash_examples": "316f54894dcb6f0d",
      "hash_full_prompts": "ba7f9b257c923f55",
      "hash_input_tokens": "8de2f010e9daf7fe",
      "hash_cont_tokens": "1f06ad3e4b80f7cc"
    },
    "truncated": 0,
    "non_truncated": 0,
    "padded": 0,
    "non_padded": 0
  },
  "pyarrow_version": "26.0.0",
  "python_version": "3.12.14 (main, Aug 25 2026, 14:01:42) [MSC v.1944 64 bit (AMD64)]",
  "aggregation": "Descriptive metric-specific task macro means and sample-weighted means; no official full-SK-bench overall score.",
  "limitations": [
    "Partial available zero-shot subset: 222 of 412 task variants.",
    "Only the validation-selected LR=0.00010 was benchmarked.",
    "Sample counts are evaluation instances; prompt variants reuse underlying questions.",
    "One console warning reports context truncation; all DETAILS counters are zero, so the recorded counters do not reliably enumerate affected inputs.",
    "The audit cannot recover the number of affected tasks/samples or quantify output generation-size termination.",
    "All-zero summary counters include non_truncated=0 and cannot prove absence of truncation.",
    "Raw DETAILS and generation-cache parquets are excluded from this curated tree."
  ]
}