changes

titan_aya_225k/config.yaml

@@ -0,0 +1,67 @@
+stage: sft
+do_train: true
+do_eval: true
+
+model_name_or_path: slovak-nlp/mistral-sk-7b
+trust_remote_code: true
+
+dataset_dir: data
+dataset: aya_slovak_full
+template: alpaca
+cutoff_len: 1024
+packing: false
+train_on_prompt: false
+
+finetuning_type: lora
+lora_rank: 16
+lora_alpha: 16
+lora_dropout: 0.0
+lora_target: q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj
+
+per_device_train_batch_size: 1
+per_device_eval_batch_size: 1
+gradient_accumulation_steps: 8
+
+learning_rate: 0.00005
+lr_scheduler_type: cosine
+warmup_steps: 10000
+max_grad_norm: 0.5
+
+eval_strategy: steps
+val_size: 1000
+eval_steps: 25000
+
+fp16: true
+optim: adamw_torch
+flash_attn: auto
+plot_loss: true
+
+ddp_timeout: 180000000
+ddp_find_unused_parameters: false
+include_num_input_tokens_seen: true
+
+enable_thinking: false
+
+use_unsloth: true
+use_unsloth_gc: true
+
+streaming: true
+buffer_size: 50000
+
+max_steps: 225000
+
+overwrite_cache: false
+
+output_dir: /home/schwarc/diplomovka/aya_streaming_lora_unsloth_2gpu_dispatchfalse_225k
+overwrite_output_dir: false
+
+logging_steps: 50
+save_steps: 25000
+save_total_limit: 2
+report_to: none
+
+seed: 42
+
+accelerator_config:
+  dispatch_batches: false
+  split_batches: false