diff --git a/model/supervised_finetuning/.gitignore b/model/supervised_finetuning/.gitignore new file mode 100644 index 00000000..16d3c4db --- /dev/null +++ b/model/supervised_finetuning/.gitignore @@ -0,0 +1 @@ +.cache diff --git a/model/supervised_finetuning/configs/config.yaml b/model/supervised_finetuning/configs/config.yaml index 63a2a592..69f30fe3 100644 --- a/model/supervised_finetuning/configs/config.yaml +++ b/model/supervised_finetuning/configs/config.yaml @@ -13,6 +13,7 @@ defaults: logging_steps: 10 max_grad_norm: 2.0 save_total_limit: 4 + fp16: false eval_accumulation_steps: freeze_layer: datasets: diff --git a/model/supervised_finetuning/custom_datasets/prompt_dialogue.py b/model/supervised_finetuning/custom_datasets/prompt_dialogue.py index d7b80761..4a2a0be5 100644 --- a/model/supervised_finetuning/custom_datasets/prompt_dialogue.py +++ b/model/supervised_finetuning/custom_datasets/prompt_dialogue.py @@ -20,7 +20,7 @@ class OAPrivate(Dataset): total_prob = reduce(lambda prev, split: prev + split[1], self.splits.items(), 0) assert math.isclose(total_prob, 1), "Make sure OAPrivate split ratios add to 1" - jsonl_file = os.path.join(data_path, self.file) + jsonl_file = os.path.join(data_path, file) with open(jsonl_file, "r", encoding="utf-8") as f: lines = f.readlines() diff --git a/model/supervised_finetuning/trainer.py b/model/supervised_finetuning/trainer.py index 48847862..cb5efa5e 100644 --- a/model/supervised_finetuning/trainer.py +++ b/model/supervised_finetuning/trainer.py @@ -217,7 +217,7 @@ if __name__ == "__main__": learning_rate=float(training_conf.learning_rate), deepspeed="configs/zero_config.json" if training_conf.deepspeed else None, optim=optimizer, - fp16=True, + fp16=training_conf.fp16, local_rank=training_conf.local_rank, gradient_checkpointing=training_conf.gradient_checkpointing, gradient_accumulation_steps=training_conf.gradient_accumulation_steps,