merge deepspeed

model/supervised_finetuning/configs/config.yaml

@@ -6,7 +6,7 @@ defaults:
   per_device_eval_batch_size: 2
   weight_decay: 0.00
   warmup_steps: 600
-  eval_steps: 200
+  eval_steps: 100
   save_steps: 500
   max_length: 512
   num_train_epochs: 3
@@ -17,6 +17,7 @@ defaults:
   freeze_layer:
   datasets:
     - webgpt
+    - prompt_dialogue
   cache_dir: ~/.cache
   loss_fn: CrossEntropyLoss
   eval_size:
@@ -44,11 +45,21 @@ gpt-jt:
   per_device_train_batch_size: 4
   per_device_eval_batch_size: 4
 
+codegen:
+  learning_rate: 2e-6
+  model_name: Salesforce/codegen-2B-multi
+  weight_decay: 0.01
+  max_length: 812
+  warmup_steps: 600
+  gradient_checkpointing: false
+  gradient_accumulation_steps: 5
+  per_device_train_batch_size: 4
+  per_device_eval_batch_size: 4
+
 debug:
   eval_steps: 20
   eval_size: 100
-  model_name: EleutherAI/gpt-j-6B
-  gradient_accumulation_steps: 2
+  gradient_accumulation_steps: 1
   per_device_train_batch_size: 1
   per_device_eval_batch_size: 1
-  quantization: 8bit
+  quantization: