Upload training_config.yml with huggingface_hub

training_config.yml

@@ -5,30 +5,31 @@ model:
  - v_proj
  apply_lora_to_mlp: false
  apply_lora_to_output: false
- lora_rank: 8
- lora_alpha: 16
+ lora_rank: 32
+ lora_alpha: 64
  perception_tokens: 2
  use_clip: false
 tokenizer:
  _component_: models.a2a_tokenizer
- path: checkpoints/Meta-Llama-3-8B-Instruct/original/tokenizer.model
+ path: models/tokenizer.model
 checkpointer:
  _component_: torchtune.utils.FullModelMetaCheckpointer
- checkpoint_dir: checkpoints/Meta-Llama-3-8B-Instruct/original
+ checkpoint_dir: 
  checkpoint_files:
- - consolidated.02.pth
+ - 
  adapter_checkpoint: null
  recipe_checkpoint: null
- output_dir: output_checkpoints/experiment_3
+ output_dir: output_checkpoints/experiment_1
  model_type: LLAMA3
 resume_from_checkpoint: false
-interim_checkpoint_steps: 5000
+interim_checkpoint_steps: 15000
 interim_gen_steps: null
-max_new_tokens: 100
-temperature: 0.6
-top_k: 300
+max_new_tokens: 88
+temperature: 0.7
+top_k: 232
 dataset:
  _component_: ds.EvenBatcher
+ buffer_size: 73
  dataset:
  _component_: ds.RoundRobinDataset
  datasets:
@@ -45,21 +46,21 @@ dataset:
  train_on_input: false
 seed: null
 shuffle: true
-batch_size: 8
+batch_size: 6
 optimizer:
  _component_: torch.optim.AdamW
- weight_decay: 0.01
- lr: 0.0003
+ weight_decay: 0.99
+ lr: 20.0e-05
 lr_scheduler:
  _component_: torchtune.modules.get_cosine_schedule_with_warmup
- num_warmup_steps: 100
+ num_warmup_steps: 4
 loss:
  _component_: torch.nn.CrossEntropyLoss
-epochs: 2
-max_steps_per_epoch: 10
-gradient_accumulation_steps: 64
+epochs: 60
+max_steps_per_epoch: null
+gradient_accumulation_steps: 260
 compile: false
-output_dir: /tmp/lora_finetune_output
+output_dir: /workspace/hebbanvogola/lora_finetune_output
 metric_logger:
  _component_: torchtune.utils.metric_logging.DiskLogger
  log_dir: ${output_dir}
@@ -76,8 +77,8 @@ inference:
  {video}
 
  Caption the previous video.'
- max_new_tokens: 300
- temperature: 0.6
- top_k: 300
+ max_new_tokens: 231
+ temperature: 0.8
+ top_k: 231
  quantizer: null
 gradient-accumulation-steps: 32