# Jeden trening tohto modelu: 4 H200, DDP, Unsloth a bf16.
model_name_or_path: "slovak-nlp/mistral-sk-7b"
model_revision: "089497ae0d72018e9591895c54f774bf0b4a83a4"
stage: sft
do_train: true
do_eval: true
finetuning_type: lora
template: alpaca
use_unsloth: true
use_unsloth_gc: true
bf16: true
infer_dtype: bfloat16
flash_attn: sdpa

dataset_dir: data/aya_slovak_fixed
dataset: aya_train
eval_dataset: aya_eval
cutoff_len: 4096
packing: false
train_on_prompt: false
preprocessing_num_workers: 8
dataloader_num_workers: 2
overwrite_cache: false

lora_rank: 16
lora_alpha: 32
lora_dropout: 0.0
lora_target: q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj

per_device_train_batch_size: 8
per_device_eval_batch_size: 8
gradient_accumulation_steps: 1
learning_rate: 0.0001
num_train_epochs: 1.0
lr_scheduler_type: cosine
warmup_ratio: 0.08
optim: adamw_torch
weight_decay: 0.0
max_grad_norm: 0.5
gradient_checkpointing: true
ddp_find_unused_parameters: false

output_dir: runs/mistral/adapter
overwrite_output_dir: false
logging_steps: 10
eval_strategy: steps
eval_steps: 1000
save_strategy: steps
save_steps: 1000
save_total_limit: 2
save_only_model: false
load_best_model_at_end: true
metric_for_best_model: loss
greater_is_better: false
predict_with_generate: false
plot_loss: true
report_to: none
seed: 42
data_seed: 42