diff --git a/titan_aya_1gpu_unstable_run/train_mistral_sk_aya_slovak_full_lora.yaml b/titan_aya_1gpu_unstable_run/train_mistral_sk_aya_slovak_full_lora.yaml new file mode 100644 index 0000000..10e0e6d --- /dev/null +++ b/titan_aya_1gpu_unstable_run/train_mistral_sk_aya_slovak_full_lora.yaml @@ -0,0 +1,63 @@ +stage: sft +do_train: true + +model_name_or_path: slovak-nlp/mistral-sk-7b +trust_remote_code: true + +dataset_dir: data +dataset: aya_slovak_full +template: alpaca +cutoff_len: 1024 + +finetuning_type: lora +lora_rank: 16 +lora_alpha: 32 +lora_dropout: 0.05 +lora_target: q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj + + +per_device_train_batch_size: 1 +per_device_eval_batch_size: 1 +gradient_accumulation_steps: 4 + +learning_rate: 0.0002 +lr_scheduler_type: cosine +warmup_steps: 500 +max_grad_norm: 1.0 + +eval_strategy: steps + +fp16: true +optim: adamw_8bit +flash_attn: auto +plot_loss: true + +ddp_timeout: 180000000 +ddp_find_unused_parameters: false +include_num_input_tokens_seen: true + +enable_thinking: false + + +use_unsloth: true +use_unsloth_gc: true + +# Streaming mode - dataset sa nebude celý tokenizovať do cache naraz. +streaming: true +buffer_size: 16384 + +# Približne jeden prechod cez celý slovenský Aya subset: +# 4 148 187 príkladov / effective batch 4 ≈ 1 037 000 krokov. +max_steps: 1037000 + +val_size: 1000 +overwrite_cache: false + +output_dir: /home/schwarc/diplomovka/mistral_sk_aya_slovak_full_lora_unsloth_streaming_1gpu +overwrite_output_dir: false + +logging_steps: 50 +save_steps: 10000 +save_total_limit: 2 +eval_steps: 50000 +report_to: none