stage: sft
do_train: true
do_eval: true

model_name_or_path: slovak-nlp/mistral-sk-7b
trust_remote_code: true

dataset_dir: data
dataset: aya_slovak_full
template: alpaca
cutoff_len: 1024
packing: false
train_on_prompt: false

finetuning_type: lora
lora_rank: 16
lora_alpha: 16
lora_dropout: 0.0
lora_target: q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj

per_device_train_batch_size: 1
per_device_eval_batch_size: 1
gradient_accumulation_steps: 8

learning_rate: 0.00005
lr_scheduler_type: cosine
warmup_steps: 10000
max_grad_norm: 0.5

eval_strategy: steps
val_size: 1000
eval_steps: 25000

fp16: true
optim: adamw_torch
flash_attn: auto
plot_loss: true

ddp_timeout: 180000000
ddp_find_unused_parameters: false
include_num_input_tokens_seen: true

enable_thinking: false

use_unsloth: true
use_unsloth_gc: true

streaming: true
buffer_size: 50000

max_steps: 225000

overwrite_cache: false

output_dir: /home/schwarc/diplomovka/aya_streaming_lora_unsloth_2gpu_dispatchfalse_225k
overwrite_output_dir: false

logging_steps: 50
save_steps: 25000
save_total_limit: 2
report_to: none

seed: 42

accelerator_config:
  dispatch_batches: false
  split_batches: false