stage: sft
do_train: true
do_eval: true
model_name_or_path: slovak-nlp/mistral-sk-7b
trust_remote_code: true
dataset_dir: data
dataset: aya_slovak_hpo_30k
template: alpaca
cutoff_len: 1024
packing: false
train_on_prompt: false
finetuning_type: lora
lora_rank: 16
lora_alpha: 32
lora_dropout: 0.0
lora_target: q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj
per_device_train_batch_size: 1
per_device_eval_batch_size: 1
gradient_accumulation_steps: 16
learning_rate: 0.00018690446893209942
lr_scheduler_type: cosine
warmup_steps: 150
max_grad_norm: 0.5
bf16: true
fp16: false
optim: adamw_torch
flash_attn: auto
plot_loss: true
ddp_timeout: 180000000
ddp_find_unused_parameters: false
include_num_input_tokens_seen: true
enable_thinking: false
use_unsloth: true
use_unsloth_gc: true
streaming: false
eval_strategy: steps
val_size: 1000
eval_steps: 300
overwrite_cache: false
overwrite_output_dir: true
output_dir: /home/schwarc/diplomovka/optuna_lr_hpo_1gpu/trials/trial_004_lr_1.869e-04
logging_steps: 10
save_steps: 1800
save_total_limit: 1
report_to: none
seed: 42
accelerator_config:
  dispatch_batches: false
  split_batches: false
num_train_epochs: 1.0