From f304921bd59ab4af5d664215533ac880770f0b44 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Szymon=20O=C5=BC=C3=B3g?=
 <58388001+SzymonOzog@users.noreply.github.com>
Date: Sat, 7 Jan 2023 16:36:55 +0100
Subject: [PATCH] Added deberta configs

---
 .../instructor/configs/deberta-v2-xlarge.yml      | 15 +++++++++++++++
 .../reward/instructor/configs/deberta-v3-base.yml | 14 ++++++++++++++
 .../configs/deberta-v3-large-squad2.yml           | 13 +++++++++++++
 .../instructor/configs/deberta-v3-large.yml       | 14 ++++++++++++++
 4 files changed, 56 insertions(+)
 create mode 100644 model/reward/instructor/configs/deberta-v2-xlarge.yml
 create mode 100644 model/reward/instructor/configs/deberta-v3-base.yml
 create mode 100644 model/reward/instructor/configs/deberta-v3-large-squad2.yml
 create mode 100644 model/reward/instructor/configs/deberta-v3-large.yml

diff --git a/model/reward/instructor/configs/deberta-v2-xlarge.yml b/model/reward/instructor/configs/deberta-v2-xlarge.yml
new file mode 100644
index 00000000..8bc4a182
--- /dev/null
+++ b/model/reward/instructor/configs/deberta-v2-xlarge.yml
@@ -0,0 +1,15 @@
+model_name: microsoft/deberta-v2-xlarge
+learning_rate: 1e-5
+freeze_layer: 15
+scheduler: cosine
+gradient_checkpointing: false
+gradient_accumulation_steps: 16
+per_device_train_batch_size: 1
+warmup_steps: 600
+eval_steps: 200
+save_steps: 500
+max_length: 512
+num_train_epochs: 2
+datasets:
+  - webgpt
+  - hfsummary
diff --git a/model/reward/instructor/configs/deberta-v3-base.yml b/model/reward/instructor/configs/deberta-v3-base.yml
new file mode 100644
index 00000000..7023709c
--- /dev/null
+++ b/model/reward/instructor/configs/deberta-v3-base.yml
@@ -0,0 +1,14 @@
+model_name: microsoft/deberta-v3-base
+learning_rate: 1e-5
+scheduler: cosine
+gradient_checkpointing: false
+gradient_accumulation_steps: 32
+per_device_train_batch_size: 2
+warmup_steps: 600
+eval_steps: 200
+save_steps: 500
+max_length: 512
+num_train_epochs: 2
+datasets:
+  - webgpt
+  - hfsummary
diff --git a/model/reward/instructor/configs/deberta-v3-large-squad2.yml b/model/reward/instructor/configs/deberta-v3-large-squad2.yml
new file mode 100644
index 00000000..47275309
--- /dev/null
+++ b/model/reward/instructor/configs/deberta-v3-large-squad2.yml
@@ -0,0 +1,13 @@
+model_name: deepset/deberta-v3-large-squad2
+learning_rate: 1e-5
+gradient_checkpointing: false
+gradient_accumulation_steps: 32
+per_device_train_batch_size: 1
+warmup_steps: 600
+eval_steps: 200
+save_steps: 500
+max_length: 512
+num_train_epochs: 2
+datasets:
+  - webgpt
+  - hfsummary
diff --git a/model/reward/instructor/configs/deberta-v3-large.yml b/model/reward/instructor/configs/deberta-v3-large.yml
new file mode 100644
index 00000000..0a910408
--- /dev/null
+++ b/model/reward/instructor/configs/deberta-v3-large.yml
@@ -0,0 +1,14 @@
+model_name: microsoft/deberta-v3-large
+learning_rate: 1e-5
+scheduler: cosine
+gradient_checkpointing: false
+gradient_accumulation_steps: 32
+per_device_train_batch_size: 1
+warmup_steps: 600
+eval_steps: 200
+save_steps: 500
+max_length: 512
+num_train_epochs: 2
+datasets:
+  - webgpt
+  - hfsummary