Config_add

titan_aya_1gpu_unstable_run/train_mistral_sk_aya_slovak_full_lora.yaml

@@ -0,0 +1,63 @@
+stage: sft
+do_train: true
+
+model_name_or_path: slovak-nlp/mistral-sk-7b
+trust_remote_code: true
+
+dataset_dir: data
+dataset: aya_slovak_full
+template: alpaca
+cutoff_len: 1024
+
+finetuning_type: lora
+lora_rank: 16
+lora_alpha: 32
+lora_dropout: 0.05
+lora_target: q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj
+
+
+per_device_train_batch_size: 1
+per_device_eval_batch_size: 1
+gradient_accumulation_steps: 4
+
+learning_rate: 0.0002
+lr_scheduler_type: cosine
+warmup_steps: 500
+max_grad_norm: 1.0
+
+eval_strategy: steps
+
+fp16: true
+optim: adamw_8bit
+flash_attn: auto
+plot_loss: true
+
+ddp_timeout: 180000000
+ddp_find_unused_parameters: false
+include_num_input_tokens_seen: true
+
+enable_thinking: false
+
+
+use_unsloth: true
+use_unsloth_gc: true
+
+# Streaming mode - dataset sa nebude celý tokenizovať do cache naraz.
+streaming: true
+buffer_size: 16384
+
+# Približne jeden prechod cez celý slovenský Aya subset:
+# 4 148 187 príkladov / effective batch 4 ≈ 1 037 000 krokov.
+max_steps: 1037000
+
+val_size: 1000
+overwrite_cache: false
+
+output_dir: /home/schwarc/diplomovka/mistral_sk_aya_slovak_full_lora_unsloth_streaming_1gpu
+overwrite_output_dir: false
+
+logging_steps: 50
+save_steps: 10000
+save_total_limit: 2
+eval_steps: 50000
+report_to: none