diff --git a/model/supervised_finetuning/README.md b/model/supervised_finetuning/README.md index 387e91e4..1a3af828 100644 --- a/model/supervised_finetuning/README.md +++ b/model/supervised_finetuning/README.md @@ -66,6 +66,17 @@ different 500 examples from `prompt_dialogue`. This works with `torch.distributed`. +## Training only on OA internal data: + +To experiment with the Open Assistant data simply run: + +```bash +python trainer.py --configs defaults oa_dataset_only galactica-125m +``` + +Change the `data_path` in the `oa_dataset_only` from the `configs/config.yaml` +file to the correct path. + ## Model Normally you should be able to add new models in `configs/config.yml` diff --git a/model/supervised_finetuning/configs/config.yaml b/model/supervised_finetuning/configs/config.yaml index 92a1a793..63a2a592 100644 --- a/model/supervised_finetuning/configs/config.yaml +++ b/model/supervised_finetuning/configs/config.yaml @@ -17,13 +17,12 @@ defaults: freeze_layer: datasets: - webgpt - # - prompt_dialogue - squad_v2 - adversarial_qa - trivia_qa_nocontext - xsum - cnn_dailymail - - prompt_dialogue + - prompt_dialogue # TODO: need to fix the url - multi_news - scitldr - soda @@ -49,12 +48,16 @@ defaults: fuse_gelu: true log_wandb: true samples_mixing: false # uses collator that mixes samples in the batch to create a single sample with possible multiple tasks within + verbose: false oa_dataset_only: datasets: - - oa_pricate: - data_path: .cache - val_split: 0.0 + - oa_private: + data_path: .cache + split: sft + val_split: 0.0 + fraction: 1 + file: 2023-02-10_oasst_prod.jsonl galactica-125m: learning_rate: 5e-5 @@ -89,6 +92,7 @@ codegen: per_device_eval_batch_size: 4 debug: + model_name: EleutherAI/pythia-70m-deduped eval_steps: 20 eval_size: 20 gradient_accumulation_steps: 1 @@ -96,3 +100,4 @@ debug: per_device_eval_batch_size: 1 quantization: false log_wandb: false + verbose: true