From 5a43a3a321b0f877a8de91ec34ec22ab5bad1aee Mon Sep 17 00:00:00 2001
From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com>
Date: Mon, 6 Feb 2023 18:57:13 +0530
Subject: [PATCH] seq cls examples update

---
 examples/sequence_classification/LoRA.ipynb   | 1182 ++++-------------
 .../sequence_classification/P_Tuning.ipynb    | 1070 +++------------
 .../Prompt_Tuning.ipynb                       |  923 +++----------
 .../prefix_tuning.ipynb                       |  940 ++++---------
 src/peft/peft_model.py                        |   16 +-
 5 files changed, 903 insertions(+), 3228 deletions(-)

diff --git a/examples/sequence_classification/LoRA.ipynb b/examples/sequence_classification/LoRA.ipynb
index 018fbd9..e9dd6da 100644
--- a/examples/sequence_classification/LoRA.ipynb
+++ b/examples/sequence_classification/LoRA.ipynb
@@ -5,7 +5,23 @@
    "execution_count": 1,
    "id": "a9935ae2",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "===================================BUG REPORT===================================\n",
+      "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
+      "For effortless bug reporting copy-paste your error into this form: https://docs.google.com/forms/d/e/1FAIpQLScPB8emS3Thkp66nvqwmjTEgxp8Y9ufuWTzFyr9kJ5AoI47dQ/viewform?usp=sf_link\n",
+      "================================================================================\n",
+      "CUDA SETUP: CUDA runtime path found: /home/sourab/miniconda3/envs/ml/lib/libcudart.so\n",
+      "CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n",
+      "CUDA SETUP: Detected CUDA version 117\n",
+      "CUDA SETUP: Loading binary /home/sourab/miniconda3/envs/ml/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n"
+     ]
+    }
+   ],
    "source": [
     "import argparse\n",
     "import os\n",
@@ -60,6 +76,20 @@
    "id": "c2697d07",
    "metadata": {},
    "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0f74797387a941cbb0709487b8808eba",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading readme:   0%|          | 0.00/27.9k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
     {
      "name": "stderr",
      "output_type": "stream",
@@ -70,7 +100,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "2036db72194544e89d5c81c17aa44037",
+       "model_id": "1a9ecc2f624343c3af8d1824afb66ac5",
        "version_major": 2,
        "version_minor": 0
       },
@@ -82,13 +112,46 @@
      "output_type": "display_data"
     },
     {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-121b991f592093a4.arrow\n",
-      "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-6c55b9fb8fbb12c7.arrow\n",
-      "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-b9740d82185f93e5.arrow\n"
-     ]
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "33b071c0e5794cb48b38bbf68f22b49b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/4 [00:00<?, ?ba/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a977694036394d5c99adfb13c023e258",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/1 [00:00<?, ?ba/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "facc8d9092dc4abe9e553fc8e5b795b8",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/2 [00:00<?, ?ba/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
     }
    ],
    "source": [
@@ -133,878 +196,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "id": "2ed5ac74",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Some weights of the model checkpoint at roberta-large were not used when initializing RobertaForSequenceClassification: ['lm_head.decoder.weight', 'lm_head.layer_norm.bias', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.weight', 'lm_head.dense.weight', 'lm_head.dense.bias', 'roberta.pooler.dense.bias', 'lm_head.bias']\n",
-      "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
-      "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
-      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-large and are newly initialized: ['classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.dense.bias']\n",
-      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "trainable params: 1838082 || all params: 356148226 || trainable%: 0.5161002823582785\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "PETModelForSequenceClassification(\n",
-       "  (base_model): LoRAModel(\n",
-       "    (model): RobertaForSequenceClassification(\n",
-       "      (roberta): RobertaModel(\n",
-       "        (embeddings): RobertaEmbeddings(\n",
-       "          (word_embeddings): Embedding(50265, 1024, padding_idx=1)\n",
-       "          (position_embeddings): Embedding(514, 1024, padding_idx=1)\n",
-       "          (token_type_embeddings): Embedding(1, 1024)\n",
-       "          (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "          (dropout): Dropout(p=0.1, inplace=False)\n",
-       "        )\n",
-       "        (encoder): RobertaEncoder(\n",
-       "          (layer): ModuleList(\n",
-       "            (0): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (1): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (2): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (3): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (4): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (5): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (6): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (7): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (8): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (9): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (10): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (11): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (12): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (13): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (14): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (15): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (16): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (17): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (18): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (19): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (20): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (21): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (22): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (23): RobertaLayer(\n",
-       "              (attention): RobertaAttention(\n",
-       "                (self): RobertaSelfAttention(\n",
-       "                  (query): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (value): Linear(\n",
-       "                    in_features=1024, out_features=1024, bias=True\n",
-       "                    (lora_dropout): Dropout(p=0.1, inplace=False)\n",
-       "                    (lora_A): Linear(in_features=1024, out_features=8, bias=False)\n",
-       "                    (lora_B): Linear(in_features=8, out_features=1024, bias=False)\n",
-       "                  )\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "                (output): RobertaSelfOutput(\n",
-       "                  (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                  (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
-       "                )\n",
-       "              )\n",
-       "              (intermediate): RobertaIntermediate(\n",
-       "                (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "                (intermediate_act_fn): GELUActivation()\n",
-       "              )\n",
-       "              (output): RobertaOutput(\n",
-       "                (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "          )\n",
-       "        )\n",
-       "      )\n",
-       "      (classifier): RobertaClassificationHead(\n",
-       "        (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "        (dropout): Dropout(p=0.1, inplace=False)\n",
-       "        (out_proj): Linear(in_features=1024, out_features=2, bias=True)\n",
-       "      )\n",
-       "    )\n",
-       "  )\n",
-       ")"
-      ]
-     },
-     "execution_count": 5,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "model = AutoModelForSequenceClassification.from_pretrained(model_name_or_path, return_dict=True)\n",
     "model = get_peft_model(model, peft_config)\n",
@@ -1039,301 +234,301 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "  0%|                                                                       | 0/115 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:29<00:00,  3.88it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.62it/s]\n"
+      "  0%|                                                                                                  | 0/115 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:28<00:00,  4.08it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.68it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 0: {'accuracy': 0.7205882352941176, 'f1': 0.8283132530120482}\n"
+      "epoch 0: {'accuracy': 0.7009803921568627, 'f1': 0.8189910979228486}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.19it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.63it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.18it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.64it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 1: {'accuracy': 0.8137254901960784, 'f1': 0.8737541528239203}\n"
+      "epoch 1: {'accuracy': 0.7622549019607843, 'f1': 0.8482003129890453}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.19it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.62it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.20it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.63it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 2: {'accuracy': 0.8627450980392157, 'f1': 0.9003558718861211}\n"
+      "epoch 2: {'accuracy': 0.8651960784313726, 'f1': 0.9005424954792043}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.18it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.62it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.21it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.62it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 3: {'accuracy': 0.8700980392156863, 'f1': 0.9065255731922399}\n"
+      "epoch 3: {'accuracy': 0.8921568627450981, 'f1': 0.9228070175438596}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.19it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.62it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.20it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.62it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 4: {'accuracy': 0.8946078431372549, 'f1': 0.9238938053097344}\n"
+      "epoch 4: {'accuracy': 0.8970588235294118, 'f1': 0.9257950530035336}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.19it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.62it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.16it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.01it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 5: {'accuracy': 0.8799019607843137, 'f1': 0.9135802469135803}\n"
+      "epoch 5: {'accuracy': 0.8823529411764706, 'f1': 0.9169550173010381}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.17it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.60it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:30<00:00,  3.81it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.62it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 6: {'accuracy': 0.8897058823529411, 'f1': 0.9192100538599641}\n"
+      "epoch 6: {'accuracy': 0.8799019607843137, 'f1': 0.9170896785109983}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.19it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.61it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.16it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.61it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 7: {'accuracy': 0.875, 'f1': 0.9103690685413005}\n"
+      "epoch 7: {'accuracy': 0.8799019607843137, 'f1': 0.9150779896013865}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.14it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.19it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.18it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.61it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 8: {'accuracy': 0.8799019607843137, 'f1': 0.9123434704830053}\n"
+      "epoch 8: {'accuracy': 0.8921568627450981, 'f1': 0.9233449477351917}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:52<00:00,  2.20it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:03<00:00,  3.92it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.18it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.59it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 9: {'accuracy': 0.8946078431372549, 'f1': 0.9230769230769231}\n"
+      "epoch 9: {'accuracy': 0.8872549019607843, 'f1': 0.9217687074829931}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [01:00<00:00,  1.92it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:03<00:00,  3.94it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.16it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.61it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 10: {'accuracy': 0.875, 'f1': 0.9109947643979057}\n"
+      "epoch 10: {'accuracy': 0.8774509803921569, 'f1': 0.9137931034482758}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:59<00:00,  1.93it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:03<00:00,  3.94it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:29<00:00,  3.90it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.81it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 11: {'accuracy': 0.8946078431372549, 'f1': 0.9236234458259325}\n"
+      "epoch 11: {'accuracy': 0.9068627450980392, 'f1': 0.9321428571428573}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:59<00:00,  1.92it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:03<00:00,  3.88it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:28<00:00,  4.05it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.59it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 12: {'accuracy': 0.8872549019607843, 'f1': 0.9184397163120569}\n"
+      "epoch 12: {'accuracy': 0.8946078431372549, 'f1': 0.925476603119584}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [01:21<00:00,  1.41it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:06<00:00,  1.97it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.17it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.58it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 13: {'accuracy': 0.8872549019607843, 'f1': 0.9169675090252707}\n"
+      "epoch 13: {'accuracy': 0.8897058823529411, 'f1': 0.922279792746114}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [01:59<00:00,  1.04s/it]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:06<00:00,  1.98it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.18it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.61it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 14: {'accuracy': 0.8848039215686274, 'f1': 0.9168141592920355}\n"
+      "epoch 14: {'accuracy': 0.8970588235294118, 'f1': 0.9265734265734265}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [01:59<00:00,  1.04s/it]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:06<00:00,  1.99it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.18it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.60it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 15: {'accuracy': 0.8897058823529411, 'f1': 0.9209138840070299}\n"
+      "epoch 15: {'accuracy': 0.8970588235294118, 'f1': 0.9263157894736843}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [01:29<00:00,  1.29it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:03<00:00,  3.82it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.17it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.59it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 16: {'accuracy': 0.8848039215686274, 'f1': 0.917975567190227}\n"
+      "epoch 16: {'accuracy': 0.8921568627450981, 'f1': 0.9233449477351917}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [01:44<00:00,  1.10it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:06<00:00,  2.05it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.18it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.58it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 17: {'accuracy': 0.9019607843137255, 'f1': 0.9290780141843972}\n"
+      "epoch 17: {'accuracy': 0.8897058823529411, 'f1': 0.9220103986135182}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [01:59<00:00,  1.04s/it]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:06<00:00,  2.05it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:30<00:00,  3.78it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.58it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 18: {'accuracy': 0.8946078431372549, 'f1': 0.9238938053097344}\n"
+      "epoch 18: {'accuracy': 0.8921568627450981, 'f1': 0.9233449477351917}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [02:00<00:00,  1.04s/it]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:06<00:00,  2.06it/s]"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:27<00:00,  4.16it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.60it/s]"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 19: {'accuracy': 0.8946078431372549, 'f1': 0.9238938053097344}\n"
+      "epoch 19: {'accuracy': 0.8946078431372549, 'f1': 0.924693520140105}\n"
      ]
     },
     {
@@ -1373,10 +568,113 @@
     "    print(f\"epoch {epoch}:\", eval_metric)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "f2b2caca",
+   "metadata": {},
+   "source": [
+    "## Share adapters on the 🤗 Hub"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "990b3c93",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "CommitInfo(commit_url='https://huggingface.co/smangrul/roberta-large-peft-lora/commit/c2c661898b8b6a0c68ecd068931e598d0a79686b', commit_message='Upload model', commit_description='', oid='c2c661898b8b6a0c68ecd068931e598d0a79686b', pr_url=None, pr_revision=None, pr_num=None)"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model.push_to_hub(\"smangrul/roberta-large-peft-lora\", use_auth_token=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9d140b26",
+   "metadata": {},
+   "source": [
+    "## Load adapters from the Hub\n",
+    "\n",
+    "You can also directly load adapters from the Hub using the commands below:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "4d55c87d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of the model checkpoint at roberta-large were not used when initializing RobertaForSequenceClassification: ['lm_head.bias', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.decoder.weight', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.bias']\n",
+      "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+      "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
+      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-large and are newly initialized: ['classifier.dense.bias', 'classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.out_proj.weight']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
+      "  0%|                                                                                                   | 0/13 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.45it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'accuracy': 0.8946078431372549, 'f1': 0.924693520140105}\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "from peft import PeftModel, PeftConfig\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
+    "\n",
+    "peft_model_id = \"smangrul/roberta-large-peft-lora\"\n",
+    "config = PeftConfig.from_pretrained(peft_model_id)\n",
+    "inference_model = AutoModelForSequenceClassification.from_pretrained(config.base_model_name_or_path)\n",
+    "tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)\n",
+    "\n",
+    "# Load the Lora model\n",
+    "inference_model = PeftModel.from_pretrained(inference_model, peft_model_id)\n",
+    "\n",
+    "inference_model.to(device)\n",
+    "inference_model.eval()\n",
+    "for step, batch in enumerate(tqdm(eval_dataloader)):\n",
+    "    batch.to(device)\n",
+    "    with torch.no_grad():\n",
+    "        outputs = inference_model(**batch)\n",
+    "    predictions = outputs.logits.argmax(dim=-1)\n",
+    "    predictions, references = predictions, batch[\"labels\"]\n",
+    "    metric.add_batch(\n",
+    "        predictions=predictions,\n",
+    "        references=references,\n",
+    "    )\n",
+    "\n",
+    "eval_metric = metric.compute()\n",
+    "print(eval_metric)\n"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "990b3c93",
+   "id": "27c43da1",
    "metadata": {},
    "outputs": [],
    "source": []
@@ -1384,7 +682,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
diff --git a/examples/sequence_classification/P_Tuning.ipynb b/examples/sequence_classification/P_Tuning.ipynb
index f4e2528..5b9f91d 100644
--- a/examples/sequence_classification/P_Tuning.ipynb
+++ b/examples/sequence_classification/P_Tuning.ipynb
@@ -5,7 +5,23 @@
    "execution_count": 1,
    "id": "a825ba6b",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "===================================BUG REPORT===================================\n",
+      "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
+      "For effortless bug reporting copy-paste your error into this form: https://docs.google.com/forms/d/e/1FAIpQLScPB8emS3Thkp66nvqwmjTEgxp8Y9ufuWTzFyr9kJ5AoI47dQ/viewform?usp=sf_link\n",
+      "================================================================================\n",
+      "CUDA SETUP: CUDA runtime path found: /home/sourab/miniconda3/envs/ml/lib/libcudart.so\n",
+      "CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n",
+      "CUDA SETUP: Detected CUDA version 117\n",
+      "CUDA SETUP: Loading binary /home/sourab/miniconda3/envs/ml/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n"
+     ]
+    }
+   ],
    "source": [
     "import argparse\n",
     "import os\n",
@@ -34,7 +50,7 @@
     "task = \"mrpc\"\n",
     "peft_type = PeftType.P_TUNING\n",
     "device = \"cuda\"\n",
-    "num_epochs = 30"
+    "num_epochs = 20"
    ]
   },
   {
@@ -69,7 +85,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "a2e4639741674cd199ba4cd553c3fabe",
+       "model_id": "a451b90675e0451489cc6426465afa32",
        "version_major": 2,
        "version_minor": 0
       },
@@ -84,9 +100,9 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-121b991f592093a4.arrow\n",
-      "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-6c55b9fb8fbb12c7.arrow\n",
-      "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-b9740d82185f93e5.arrow\n"
+      "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-9fa7887f9eaa03ae.arrow\n",
+      "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-dc593149bbeafe80.arrow\n",
+      "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-140ebe5b70e09817.arrow\n"
      ]
     }
    ],
@@ -132,647 +148,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "id": "f6bc8144",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Some weights of the model checkpoint at roberta-large were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
-      "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
-      "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
-      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-large and are newly initialized: ['classifier.out_proj.bias', 'classifier.out_proj.weight', 'classifier.dense.bias', 'classifier.dense.weight']\n",
-      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "trainable params: 1351938 || all params: 355662082 || trainable%: 0.38011867680626127\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "PETModelForSequenceClassification(\n",
-       "  (base_model): RobertaForSequenceClassification(\n",
-       "    (roberta): RobertaModel(\n",
-       "      (embeddings): RobertaEmbeddings(\n",
-       "        (word_embeddings): Embedding(50265, 1024, padding_idx=1)\n",
-       "        (position_embeddings): Embedding(514, 1024, padding_idx=1)\n",
-       "        (token_type_embeddings): Embedding(1, 1024)\n",
-       "        (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "        (dropout): Dropout(p=0.1, inplace=False)\n",
-       "      )\n",
-       "      (encoder): RobertaEncoder(\n",
-       "        (layer): ModuleList(\n",
-       "          (0): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (1): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (2): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (3): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (4): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (5): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (6): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (7): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (8): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (9): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (10): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (11): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (12): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (13): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (14): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (15): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (16): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (17): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (18): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (19): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (20): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (21): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (22): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (23): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "        )\n",
-       "      )\n",
-       "    )\n",
-       "    (classifier): RobertaClassificationHead(\n",
-       "      (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "      (dropout): Dropout(p=0.1, inplace=False)\n",
-       "      (out_proj): Linear(in_features=1024, out_features=2, bias=True)\n",
-       "    )\n",
-       "  )\n",
-       "  (word_embeddings): Embedding(50265, 1024, padding_idx=1)\n",
-       "  (prompt_encoder): PromptEncoder(\n",
-       "    (embedding): Embedding(20, 1024)\n",
-       "    (mlp_head): Sequential(\n",
-       "      (0): Linear(in_features=1024, out_features=128, bias=True)\n",
-       "      (1): ReLU()\n",
-       "      (2): Linear(in_features=128, out_features=128, bias=True)\n",
-       "      (3): ReLU()\n",
-       "      (4): Linear(in_features=128, out_features=1024, bias=True)\n",
-       "    )\n",
-       "  )\n",
-       ")"
-      ]
-     },
-     "execution_count": 5,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "model = AutoModelForSequenceClassification.from_pretrained(model_name_or_path, return_dict=True)\n",
     "model = get_peft_model(model, peft_config)\n",
@@ -807,451 +186,301 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "  0%|                                                                       | 0/115 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:32<00:00,  3.55it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.93it/s]\n"
+      "  0%|                                                                                                  | 0/115 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:32<00:00,  3.54it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.91it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 0: {'accuracy': 0.6838235294117647, 'f1': 0.8122270742358079}\n"
+      "epoch 0: {'accuracy': 0.6985294117647058, 'f1': 0.8172362555720655}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.62it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.94it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.61it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.87it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 1: {'accuracy': 0.6911764705882353, 'f1': 0.7993630573248407}\n"
+      "epoch 1: {'accuracy': 0.6936274509803921, 'f1': 0.806201550387597}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.64it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.93it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.61it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.88it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 2: {'accuracy': 0.7230392156862745, 'f1': 0.8138385502471169}\n"
+      "epoch 2: {'accuracy': 0.7132352941176471, 'f1': 0.8224582701062216}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.62it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.91it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.61it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.87it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 3: {'accuracy': 0.6985294117647058, 'f1': 0.8177777777777778}\n"
+      "epoch 3: {'accuracy': 0.7083333333333334, 'f1': 0.8199697428139183}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.63it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.91it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.61it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.90it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 4: {'accuracy': 0.6862745098039216, 'f1': 0.8128654970760235}\n"
+      "epoch 4: {'accuracy': 0.7205882352941176, 'f1': 0.8246153846153846}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.62it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.94it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.62it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.90it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 5: {'accuracy': 0.7034313725490197, 'f1': 0.7986688851913478}\n"
+      "epoch 5: {'accuracy': 0.7009803921568627, 'f1': 0.8200589970501474}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.64it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.88it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:32<00:00,  3.59it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.89it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 6: {'accuracy': 0.6911764705882353, 'f1': 0.81524926686217}\n"
+      "epoch 6: {'accuracy': 0.7254901960784313, 'f1': 0.8292682926829268}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.63it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.92it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.60it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.86it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 7: {'accuracy': 0.7156862745098039, 'f1': 0.8237082066869301}\n"
+      "epoch 7: {'accuracy': 0.7230392156862745, 'f1': 0.8269525267993874}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.63it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.94it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:34<00:00,  3.34it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.88it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 8: {'accuracy': 0.7205882352941176, 'f1': 0.8267477203647416}\n"
+      "epoch 8: {'accuracy': 0.7254901960784313, 'f1': 0.8297872340425533}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.62it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.94it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.60it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.77it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 9: {'accuracy': 0.7303921568627451, 'f1': 0.8312883435582822}\n"
+      "epoch 9: {'accuracy': 0.7230392156862745, 'f1': 0.828006088280061}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.62it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.94it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:32<00:00,  3.58it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.88it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 10: {'accuracy': 0.7328431372549019, 'f1': 0.809106830122592}\n"
+      "epoch 10: {'accuracy': 0.7181372549019608, 'f1': 0.8183254344391785}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.62it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.91it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.60it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.87it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 11: {'accuracy': 0.7107843137254902, 'f1': 0.8071895424836601}\n"
+      "epoch 11: {'accuracy': 0.7132352941176471, 'f1': 0.803361344537815}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.64it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.92it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.59it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.85it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 12: {'accuracy': 0.7205882352941176, 'f1': 0.8235294117647058}\n"
+      "epoch 12: {'accuracy': 0.7107843137254902, 'f1': 0.8206686930091186}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.64it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.93it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:32<00:00,  3.59it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.85it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 13: {'accuracy': 0.7083333333333334, 'f1': 0.8231797919762258}\n"
+      "epoch 13: {'accuracy': 0.7181372549019608, 'f1': 0.8254931714719272}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.63it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.94it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:32<00:00,  3.59it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.87it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 14: {'accuracy': 0.7083333333333334, 'f1': 0.8058727569331159}\n"
+      "epoch 14: {'accuracy': 0.7156862745098039, 'f1': 0.8253012048192772}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.63it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.93it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:32<00:00,  3.59it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.84it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 15: {'accuracy': 0.7230392156862745, 'f1': 0.8295625942684767}\n"
+      "epoch 15: {'accuracy': 0.7230392156862745, 'f1': 0.8242612752721618}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.63it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.92it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:32<00:00,  3.49it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:02<00:00,  5.84it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 16: {'accuracy': 0.7303921568627451, 'f1': 0.828125}\n"
+      "epoch 16: {'accuracy': 0.7181372549019608, 'f1': 0.8200312989045383}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.62it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.91it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:32<00:00,  3.49it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.84it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 17: {'accuracy': 0.7230392156862745, 'f1': 0.8197767145135567}\n"
+      "epoch 17: {'accuracy': 0.7107843137254902, 'f1': 0.8217522658610272}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.60it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.93it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.60it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.88it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 18: {'accuracy': 0.7181372549019608, 'f1': 0.8238897396630934}\n"
+      "epoch 18: {'accuracy': 0.7254901960784313, 'f1': 0.8292682926829268}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.60it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.90it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.61it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.89it/s]"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 19: {'accuracy': 0.7205882352941176, 'f1': 0.8240740740740742}\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.61it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.91it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "epoch 20: {'accuracy': 0.7181372549019608, 'f1': 0.8265460030165913}\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.61it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.89it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "epoch 21: {'accuracy': 0.7156862745098039, 'f1': 0.8170347003154573}\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.62it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.93it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "epoch 22: {'accuracy': 0.7328431372549019, 'f1': 0.8250401284109149}\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.60it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.92it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "epoch 23: {'accuracy': 0.7279411764705882, 'f1': 0.8289676425269645}\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.62it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.88it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "epoch 24: {'accuracy': 0.7254901960784313, 'f1': 0.826086956521739}\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.61it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.88it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "epoch 25: {'accuracy': 0.75, 'f1': 0.8333333333333334}\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.60it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.94it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "epoch 26: {'accuracy': 0.7426470588235294, 'f1': 0.8287112561174552}\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.61it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.93it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "epoch 27: {'accuracy': 0.7303921568627451, 'f1': 0.8275862068965518}\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.62it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.91it/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "epoch 28: {'accuracy': 0.7303921568627451, 'f1': 0.8259493670886074}\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:31<00:00,  3.62it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  6.93it/s]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "epoch 29: {'accuracy': 0.7303921568627451, 'f1': 0.8259493670886074}\n"
+      "epoch 19: {'accuracy': 0.7107843137254902, 'f1': 0.8206686930091186}\n"
      ]
     },
     {
@@ -1292,17 +521,132 @@
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "afaf42dd",
+   "cell_type": "markdown",
+   "id": "a43bd9fb",
    "metadata": {},
-   "outputs": [],
-   "source": []
+   "source": [
+    "## Share adapters on the 🤗 Hub"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "871b75aa",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "CommitInfo(commit_url='https://huggingface.co/smangrul/roberta-large-peft-p-tuning/commit/fa7abe613f498c76df5e16c85d9c19c3019587a7', commit_message='Upload model', commit_description='', oid='fa7abe613f498c76df5e16c85d9c19c3019587a7', pr_url=None, pr_revision=None, pr_num=None)"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model.push_to_hub(\"smangrul/roberta-large-peft-p-tuning\", use_auth_token=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1c6a9036",
+   "metadata": {},
+   "source": [
+    "## Load adapters from the Hub\n",
+    "\n",
+    "You can also directly load adapters from the Hub using the commands below:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "91b0b8f5",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of the model checkpoint at roberta-large were not used when initializing RobertaForSequenceClassification: ['lm_head.decoder.weight', 'lm_head.layer_norm.bias', 'lm_head.dense.bias', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.bias', 'lm_head.dense.weight', 'lm_head.bias']\n",
+      "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+      "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
+      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-large and are newly initialized: ['classifier.dense.weight', 'classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.out_proj.bias']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e650799d58ec4bd1b21b6bc28ddf2069",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading:   0%|          | 0.00/4.29M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "  0%|                                                                                                   | 0/13 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  7.18it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'accuracy': 0.7107843137254902, 'f1': 0.8206686930091186}\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "from peft import PeftModel, PeftConfig\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
+    "\n",
+    "peft_model_id = \"smangrul/roberta-large-peft-p-tuning\"\n",
+    "config = PeftConfig.from_pretrained(peft_model_id)\n",
+    "inference_model = AutoModelForSequenceClassification.from_pretrained(config.base_model_name_or_path)\n",
+    "tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)\n",
+    "\n",
+    "# Load the Lora model\n",
+    "inference_model = PeftModel.from_pretrained(inference_model, peft_model_id)\n",
+    "\n",
+    "inference_model.to(device)\n",
+    "inference_model.eval()\n",
+    "for step, batch in enumerate(tqdm(eval_dataloader)):\n",
+    "    batch.to(device)\n",
+    "    with torch.no_grad():\n",
+    "        outputs = inference_model(**batch)\n",
+    "    predictions = outputs.logits.argmax(dim=-1)\n",
+    "    predictions, references = predictions, batch[\"labels\"]\n",
+    "    metric.add_batch(\n",
+    "        predictions=predictions,\n",
+    "        references=references,\n",
+    "    )\n",
+    "\n",
+    "eval_metric = metric.compute()\n",
+    "print(eval_metric)\n"
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "234abda3",
+   "id": "1a8d69d1",
    "metadata": {},
    "outputs": [],
    "source": []
@@ -1310,7 +654,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
diff --git a/examples/sequence_classification/Prompt_Tuning.ipynb b/examples/sequence_classification/Prompt_Tuning.ipynb
index 0c53808..ce5c068 100644
--- a/examples/sequence_classification/Prompt_Tuning.ipynb
+++ b/examples/sequence_classification/Prompt_Tuning.ipynb
@@ -5,7 +5,23 @@
    "execution_count": 1,
    "id": "9ff5004e",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "===================================BUG REPORT===================================\n",
+      "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
+      "For effortless bug reporting copy-paste your error into this form: https://docs.google.com/forms/d/e/1FAIpQLScPB8emS3Thkp66nvqwmjTEgxp8Y9ufuWTzFyr9kJ5AoI47dQ/viewform?usp=sf_link\n",
+      "================================================================================\n",
+      "CUDA SETUP: CUDA runtime path found: /home/sourab/miniconda3/envs/ml/lib/libcudart.so\n",
+      "CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n",
+      "CUDA SETUP: Detected CUDA version 117\n",
+      "CUDA SETUP: Loading binary /home/sourab/miniconda3/envs/ml/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n"
+     ]
+    }
+   ],
    "source": [
     "import argparse\n",
     "import os\n",
@@ -67,7 +83,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "ba4b9c263d024b78aed68c147f3aec63",
+       "model_id": "76198cec552441818ff107910275e5be",
        "version_major": 2,
        "version_minor": 0
       },
@@ -82,9 +98,9 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-121b991f592093a4.arrow\n",
-      "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-6c55b9fb8fbb12c7.arrow\n",
-      "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-b9740d82185f93e5.arrow\n"
+      "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-9fa7887f9eaa03ae.arrow\n",
+      "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-dc593149bbeafe80.arrow\n",
+      "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-140ebe5b70e09817.arrow\n"
      ]
     }
    ],
@@ -130,640 +146,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "id": "a3c15af0",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Some weights of the model checkpoint at roberta-large were not used when initializing RobertaForSequenceClassification: ['lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.bias', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.dense.weight', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
-      "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
-      "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
-      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-large and are newly initialized: ['classifier.dense.bias', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.weight']\n",
-      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "trainable params: 1061890 || all params: 355372034 || trainable%: 0.29881079499913604\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "PETModelForSequenceClassification(\n",
-       "  (base_model): RobertaForSequenceClassification(\n",
-       "    (roberta): RobertaModel(\n",
-       "      (embeddings): RobertaEmbeddings(\n",
-       "        (word_embeddings): Embedding(50265, 1024, padding_idx=1)\n",
-       "        (position_embeddings): Embedding(514, 1024, padding_idx=1)\n",
-       "        (token_type_embeddings): Embedding(1, 1024)\n",
-       "        (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "        (dropout): Dropout(p=0.1, inplace=False)\n",
-       "      )\n",
-       "      (encoder): RobertaEncoder(\n",
-       "        (layer): ModuleList(\n",
-       "          (0): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (1): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (2): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (3): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (4): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (5): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (6): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (7): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (8): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (9): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (10): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (11): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (12): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (13): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (14): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (15): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (16): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (17): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (18): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (19): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (20): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (21): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (22): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (23): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "        )\n",
-       "      )\n",
-       "    )\n",
-       "    (classifier): RobertaClassificationHead(\n",
-       "      (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "      (dropout): Dropout(p=0.1, inplace=False)\n",
-       "      (out_proj): Linear(in_features=1024, out_features=2, bias=True)\n",
-       "    )\n",
-       "  )\n",
-       "  (word_embeddings): Embedding(50265, 1024, padding_idx=1)\n",
-       "  (prompt_encoder): PromptEmbedding(\n",
-       "    (embedding): Embedding(10, 1024)\n",
-       "  )\n",
-       ")"
-      ]
-     },
-     "execution_count": 5,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "model = AutoModelForSequenceClassification.from_pretrained(model_name_or_path, return_dict=True)\n",
     "model = get_peft_model(model, peft_config)\n",
@@ -798,301 +184,301 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "  0%|                                                                       | 0/115 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:29<00:00,  3.92it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  7.42it/s]\n"
+      "  0%|                                                                                                  | 0/115 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [02:09<00:00,  1.13s/it]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:08<00:00,  1.62it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 0: {'accuracy': 0.6568627450980392, 'f1': 0.7727272727272726}\n"
+      "epoch 0: {'accuracy': 0.678921568627451, 'f1': 0.7956318252730109}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:28<00:00,  4.01it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  7.42it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:50<00:00,  1.04it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.22it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 1: {'accuracy': 0.6691176470588235, 'f1': 0.782608695652174}\n"
+      "epoch 1: {'accuracy': 0.696078431372549, 'f1': 0.8171091445427728}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:28<00:00,  4.01it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  7.45it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:36<00:00,  1.19it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:06<00:00,  2.00it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 2: {'accuracy': 0.6985294117647058, 'f1': 0.8188512518409425}\n"
+      "epoch 2: {'accuracy': 0.6985294117647058, 'f1': 0.8161434977578476}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:28<00:00,  4.00it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  7.47it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:37<00:00,  1.18it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:06<00:00,  2.09it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 3: {'accuracy': 0.6838235294117647, 'f1': 0.8122270742358079}\n"
+      "epoch 3: {'accuracy': 0.7058823529411765, 'f1': 0.7979797979797979}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:28<00:00,  4.00it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  7.39it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [02:03<00:00,  1.07s/it]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:07<00:00,  1.71it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 4: {'accuracy': 0.7156862745098039, 'f1': 0.8263473053892216}\n"
+      "epoch 4: {'accuracy': 0.696078431372549, 'f1': 0.8132530120481929}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:28<00:00,  4.00it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  7.36it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:53<00:00,  1.01it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.19it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 5: {'accuracy': 0.7107843137254902, 'f1': 0.8238805970149253}\n"
+      "epoch 5: {'accuracy': 0.7107843137254902, 'f1': 0.8121019108280254}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:28<00:00,  4.00it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  7.33it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:35<00:00,  1.20it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.20it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 6: {'accuracy': 0.7107843137254902, 'f1': 0.824404761904762}\n"
+      "epoch 6: {'accuracy': 0.6911764705882353, 'f1': 0.7692307692307693}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:28<00:00,  4.00it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  7.41it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:36<00:00,  1.20it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.18it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 7: {'accuracy': 0.7181372549019608, 'f1': 0.8270676691729323}\n"
+      "epoch 7: {'accuracy': 0.7156862745098039, 'f1': 0.8209876543209876}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:28<00:00,  3.99it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  7.42it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:35<00:00,  1.20it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.22it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 8: {'accuracy': 0.6911764705882353, 'f1': 0.81524926686217}\n"
+      "epoch 8: {'accuracy': 0.7205882352941176, 'f1': 0.8240740740740742}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:28<00:00,  3.99it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  7.37it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:36<00:00,  1.19it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.21it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 9: {'accuracy': 0.7352941176470589, 'f1': 0.8241042345276873}\n"
+      "epoch 9: {'accuracy': 0.7205882352941176, 'f1': 0.8229813664596273}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:28<00:00,  3.98it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  7.43it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:36<00:00,  1.20it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.35it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 10: {'accuracy': 0.7254901960784313, 'f1': 0.8276923076923076}\n"
+      "epoch 10: {'accuracy': 0.7156862745098039, 'f1': 0.8164556962025317}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:28<00:00,  3.98it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  7.43it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:35<00:00,  1.20it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.22it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 11: {'accuracy': 0.7230392156862745, 'f1': 0.8300751879699249}\n"
+      "epoch 11: {'accuracy': 0.7058823529411765, 'f1': 0.8113207547169811}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:28<00:00,  3.99it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  7.31it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:32<00:00,  1.24it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.48it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 12: {'accuracy': 0.7230392156862745, 'f1': 0.8295625942684767}\n"
+      "epoch 12: {'accuracy': 0.7009803921568627, 'f1': 0.7946127946127945}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:28<00:00,  3.98it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  7.38it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:32<00:00,  1.24it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.38it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 13: {'accuracy': 0.7132352941176471, 'f1': 0.8251121076233183}\n"
+      "epoch 13: {'accuracy': 0.7230392156862745, 'f1': 0.8186195826645265}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:28<00:00,  3.99it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  7.39it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:29<00:00,  1.29it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.31it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 14: {'accuracy': 0.7205882352941176, 'f1': 0.8267477203647416}\n"
+      "epoch 14: {'accuracy': 0.7058823529411765, 'f1': 0.8130841121495327}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:28<00:00,  3.98it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  7.41it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:30<00:00,  1.27it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.39it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 15: {'accuracy': 0.7401960784313726, 'f1': 0.8354037267080745}\n"
+      "epoch 15: {'accuracy': 0.7181372549019608, 'f1': 0.8194662480376768}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:28<00:00,  3.97it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  7.42it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:28<00:00,  1.29it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.35it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 16: {'accuracy': 0.7475490196078431, 'f1': 0.8377952755905512}\n"
+      "epoch 16: {'accuracy': 0.7254901960784313, 'f1': 0.8181818181818181}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:28<00:00,  3.99it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  7.34it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:30<00:00,  1.27it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.30it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 17: {'accuracy': 0.75, 'f1': 0.8401253918495298}\n"
+      "epoch 17: {'accuracy': 0.7205882352941176, 'f1': 0.820754716981132}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:28<00:00,  3.98it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  7.41it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:30<00:00,  1.27it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.36it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 18: {'accuracy': 0.7475490196078431, 'f1': 0.8403100775193798}\n"
+      "epoch 18: {'accuracy': 0.7254901960784313, 'f1': 0.821656050955414}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:28<00:00,  3.98it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  7.37it/s]"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:28<00:00,  1.29it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.43it/s]"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 19: {'accuracy': 0.7377450980392157, 'f1': 0.8356374807987711}\n"
+      "epoch 19: {'accuracy': 0.7303921568627451, 'f1': 0.8242811501597445}\n"
      ]
     },
     {
@@ -1133,25 +519,146 @@
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e839e449",
+   "cell_type": "markdown",
+   "id": "e1ff3f44",
    "metadata": {},
-   "outputs": [],
-   "source": []
+   "source": [
+    "## Share adapters on the 🤗 Hub"
+   ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
    "id": "0bf79cb5",
    "metadata": {},
-   "outputs": [],
-   "source": []
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "CommitInfo(commit_url='https://huggingface.co/smangrul/roberta-large-peft-prompt-tuning/commit/893a909d8499aa8778d58c781d43c3a8d9360de8', commit_message='Upload model', commit_description='', oid='893a909d8499aa8778d58c781d43c3a8d9360de8', pr_url=None, pr_revision=None, pr_num=None)"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model.push_to_hub(\"smangrul/roberta-large-peft-prompt-tuning\", use_auth_token=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "73870ad7",
+   "metadata": {},
+   "source": [
+    "## Load adapters from the Hub\n",
+    "\n",
+    "You can also directly load adapters from the Hub using the commands below:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "0654a552",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "24581bb98582444ca6114b9fa267847f",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading:   0%|          | 0.00/368 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of the model checkpoint at roberta-large were not used when initializing RobertaForSequenceClassification: ['lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'roberta.pooler.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.weight', 'lm_head.decoder.weight', 'lm_head.dense.bias']\n",
+      "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+      "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
+      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-large and are newly initialized: ['classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.dense.weight']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "f1584da4d1c54cc3873a515182674980",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading:   0%|          | 0.00/4.25M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "  0%|                                                                                                   | 0/13 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.58it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'accuracy': 0.7303921568627451, 'f1': 0.8242811501597445}\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "from peft import PeftModel, PeftConfig\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
+    "\n",
+    "peft_model_id = \"smangrul/roberta-large-peft-prompt-tuning\"\n",
+    "config = PeftConfig.from_pretrained(peft_model_id)\n",
+    "inference_model = AutoModelForSequenceClassification.from_pretrained(config.base_model_name_or_path)\n",
+    "tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)\n",
+    "\n",
+    "# Load the Lora model\n",
+    "inference_model = PeftModel.from_pretrained(inference_model, peft_model_id)\n",
+    "\n",
+    "inference_model.to(device)\n",
+    "inference_model.eval()\n",
+    "for step, batch in enumerate(tqdm(eval_dataloader)):\n",
+    "    batch.to(device)\n",
+    "    with torch.no_grad():\n",
+    "        outputs = inference_model(**batch)\n",
+    "    predictions = outputs.logits.argmax(dim=-1)\n",
+    "    predictions, references = predictions, batch[\"labels\"]\n",
+    "    metric.add_batch(\n",
+    "        predictions=predictions,\n",
+    "        references=references,\n",
+    "    )\n",
+    "\n",
+    "eval_metric = metric.compute()\n",
+    "print(eval_metric)\n"
+   ]
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -1165,7 +672,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.5 (v3.10.5:f377153967, Jun  6 2022, 12:36:10) [Clang 13.0.0 (clang-1300.0.29.30)]"
+   "version": "3.10.4"
   },
   "vscode": {
    "interpreter": {
diff --git a/examples/sequence_classification/prefix_tuning.ipynb b/examples/sequence_classification/prefix_tuning.ipynb
index b37827b..a7ca16e 100644
--- a/examples/sequence_classification/prefix_tuning.ipynb
+++ b/examples/sequence_classification/prefix_tuning.ipynb
@@ -5,7 +5,23 @@
    "execution_count": 1,
    "id": "a825ba6b",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "===================================BUG REPORT===================================\n",
+      "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
+      "For effortless bug reporting copy-paste your error into this form: https://docs.google.com/forms/d/e/1FAIpQLScPB8emS3Thkp66nvqwmjTEgxp8Y9ufuWTzFyr9kJ5AoI47dQ/viewform?usp=sf_link\n",
+      "================================================================================\n",
+      "CUDA SETUP: CUDA runtime path found: /home/sourab/miniconda3/envs/ml/lib/libcudart.so\n",
+      "CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n",
+      "CUDA SETUP: Detected CUDA version 117\n",
+      "CUDA SETUP: Loading binary /home/sourab/miniconda3/envs/ml/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n"
+     ]
+    }
+   ],
    "source": [
     "import argparse\n",
     "import os\n",
@@ -67,7 +83,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "b64e52671dd445209f5e58888db2f0d4",
+       "model_id": "be1eddbb9a7d4e6dae32fd026e167f96",
        "version_major": 2,
        "version_minor": 0
       },
@@ -82,9 +98,28 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-121b991f592093a4.arrow\n",
-      "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-6c55b9fb8fbb12c7.arrow\n",
-      "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-b9740d82185f93e5.arrow\n"
+      "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-9fa7887f9eaa03ae.arrow\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b61574844b6c499b8960fd4d78c5e549",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/1 [00:00<?, ?ba/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad/cache-7e7eacaa5160936d.arrow\n"
      ]
     }
    ],
@@ -130,640 +165,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "id": "f6bc8144",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Some weights of the model checkpoint at roberta-large were not used when initializing RobertaForSequenceClassification: ['lm_head.decoder.weight', 'lm_head.layer_norm.bias', 'lm_head.bias', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.weight', 'lm_head.dense.bias', 'roberta.pooler.dense.bias', 'lm_head.dense.weight']\n",
-      "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
-      "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
-      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-large and are newly initialized: ['classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.dense.bias']\n",
-      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "trainable params: 2034690 || all params: 356344834 || trainable%: 0.5709890549444586\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "PETModelForSequenceClassification(\n",
-       "  (base_model): RobertaForSequenceClassification(\n",
-       "    (roberta): RobertaModel(\n",
-       "      (embeddings): RobertaEmbeddings(\n",
-       "        (word_embeddings): Embedding(50265, 1024, padding_idx=1)\n",
-       "        (position_embeddings): Embedding(514, 1024, padding_idx=1)\n",
-       "        (token_type_embeddings): Embedding(1, 1024)\n",
-       "        (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "        (dropout): Dropout(p=0.1, inplace=False)\n",
-       "      )\n",
-       "      (encoder): RobertaEncoder(\n",
-       "        (layer): ModuleList(\n",
-       "          (0): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (1): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (2): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (3): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (4): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (5): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (6): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (7): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (8): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (9): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (10): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (11): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (12): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (13): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (14): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (15): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (16): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (17): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (18): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (19): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (20): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (21): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (22): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "          (23): RobertaLayer(\n",
-       "            (attention): RobertaAttention(\n",
-       "              (self): RobertaSelfAttention(\n",
-       "                (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "              (output): RobertaSelfOutput(\n",
-       "                (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "                (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "                (dropout): Dropout(p=0.1, inplace=False)\n",
-       "              )\n",
-       "            )\n",
-       "            (intermediate): RobertaIntermediate(\n",
-       "              (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
-       "              (intermediate_act_fn): GELUActivation()\n",
-       "            )\n",
-       "            (output): RobertaOutput(\n",
-       "              (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
-       "              (LayerNorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
-       "              (dropout): Dropout(p=0.1, inplace=False)\n",
-       "            )\n",
-       "          )\n",
-       "        )\n",
-       "      )\n",
-       "    )\n",
-       "    (classifier): RobertaClassificationHead(\n",
-       "      (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
-       "      (dropout): Dropout(p=0.1, inplace=False)\n",
-       "      (out_proj): Linear(in_features=1024, out_features=2, bias=True)\n",
-       "    )\n",
-       "  )\n",
-       "  (word_embeddings): Embedding(50265, 1024, padding_idx=1)\n",
-       "  (prompt_encoder): PrefixEncoder(\n",
-       "    (embedding): Embedding(20, 49152)\n",
-       "  )\n",
-       ")"
-      ]
-     },
-     "execution_count": 5,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "model = AutoModelForSequenceClassification.from_pretrained(model_name_or_path, return_dict=True)\n",
     "model = get_peft_model(model, peft_config)\n",
@@ -798,294 +203,294 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "  0%|                                                                       | 0/115 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [01:04<00:00,  1.77it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:03<00:00,  3.73it/s]\n"
+      "  0%|                                                                                                  | 0/115 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:29<00:00,  3.87it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.32it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 0: {'accuracy': 0.7181372549019608, 'f1': 0.8200312989045383}\n"
+      "epoch 0: {'accuracy': 0.7132352941176471, 'f1': 0.7876588021778584}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:58<00:00,  1.97it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:03<00:00,  3.72it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:26<00:00,  4.42it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.36it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 1: {'accuracy': 0.7720588235294118, 'f1': 0.8467874794069192}\n"
+      "epoch 1: {'accuracy': 0.6838235294117647, 'f1': 0.8122270742358079}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:58<00:00,  1.98it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:03<00:00,  3.70it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:26<00:00,  4.41it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.35it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 2: {'accuracy': 0.7990196078431373, 'f1': 0.8637873754152825}\n"
+      "epoch 2: {'accuracy': 0.8088235294117647, 'f1': 0.8717105263157895}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:58<00:00,  1.97it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:03<00:00,  3.68it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:26<00:00,  4.39it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.34it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 3: {'accuracy': 0.8259803921568627, 'f1': 0.883415435139573}\n"
+      "epoch 3: {'accuracy': 0.7549019607843137, 'f1': 0.8475609756097561}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [01:27<00:00,  1.32it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:06<00:00,  1.86it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:26<00:00,  4.37it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.34it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 4: {'accuracy': 0.8676470588235294, 'f1': 0.9}\n"
+      "epoch 4: {'accuracy': 0.8480392156862745, 'f1': 0.8938356164383561}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [01:56<00:00,  1.01s/it]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:07<00:00,  1.75it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [00:40<00:00,  2.87it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:06<00:00,  1.93it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 5: {'accuracy': 0.8578431372549019, 'f1': 0.8921933085501859}\n"
+      "epoch 5: {'accuracy': 0.8651960784313726, 'f1': 0.9053356282271946}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [01:56<00:00,  1.01s/it]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:06<00:00,  1.87it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:53<00:00,  1.01it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:07<00:00,  1.79it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 6: {'accuracy': 0.8700980392156863, 'f1': 0.9051878354203935}\n"
+      "epoch 6: {'accuracy': 0.8700980392156863, 'f1': 0.9065255731922399}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [01:22<00:00,  1.39it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:03<00:00,  3.63it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:42<00:00,  1.12it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.43it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 7: {'accuracy': 0.8774509803921569, 'f1': 0.9084249084249084}\n"
+      "epoch 7: {'accuracy': 0.8676470588235294, 'f1': 0.9042553191489361}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [01:44<00:00,  1.10it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:07<00:00,  1.74it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:27<00:00,  1.31it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.45it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 8: {'accuracy': 0.8799019607843137, 'f1': 0.9135802469135803}\n"
+      "epoch 8: {'accuracy': 0.875, 'f1': 0.9103690685413005}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [01:57<00:00,  1.02s/it]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:07<00:00,  1.73it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:29<00:00,  1.29it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.48it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 9: {'accuracy': 0.8676470588235294, 'f1': 0.9052631578947367}\n"
+      "epoch 9: {'accuracy': 0.8799019607843137, 'f1': 0.913884007029877}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [01:56<00:00,  1.01s/it]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:06<00:00,  1.87it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:43<00:00,  1.11it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:06<00:00,  1.88it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 10: {'accuracy': 0.8725490196078431, 'f1': 0.9084507042253521}\n"
+      "epoch 10: {'accuracy': 0.8725490196078431, 'f1': 0.902621722846442}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [01:26<00:00,  1.33it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:03<00:00,  3.75it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:53<00:00,  1.02it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:06<00:00,  2.02it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 11: {'accuracy': 0.8799019607843137, 'f1': 0.9113924050632911}\n"
+      "epoch 11: {'accuracy': 0.875, 'f1': 0.9090909090909091}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:54<00:00,  2.10it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.25it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:29<00:00,  1.28it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:04<00:00,  2.65it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 12: {'accuracy': 0.8651960784313726, 'f1': 0.9053356282271946}\n"
+      "epoch 12: {'accuracy': 0.8823529411764706, 'f1': 0.9139784946236559}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:26<00:00,  4.27it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:02<00:00,  6.06it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:27<00:00,  1.31it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.46it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 13: {'accuracy': 0.8774509803921569, 'f1': 0.912280701754386}\n"
+      "epoch 13: {'accuracy': 0.8602941176470589, 'f1': 0.9018932874354562}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [01:11<00:00,  1.62it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.52it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:27<00:00,  1.31it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.47it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 14: {'accuracy': 0.8799019607843137, 'f1': 0.9123434704830053}\n"
+      "epoch 14: {'accuracy': 0.8700980392156863, 'f1': 0.9075043630017452}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [01:26<00:00,  1.32it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.52it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:27<00:00,  1.31it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.49it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 15: {'accuracy': 0.8725490196078431, 'f1': 0.907473309608541}\n"
+      "epoch 15: {'accuracy': 0.875, 'f1': 0.9087656529516995}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:53<00:00,  2.14it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:01<00:00,  8.04it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:27<00:00,  1.32it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.49it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 16: {'accuracy': 0.8651960784313726, 'f1': 0.905982905982906}\n"
+      "epoch 16: {'accuracy': 0.8578431372549019, 'f1': 0.9003436426116839}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [00:51<00:00,  2.23it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:03<00:00,  3.73it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:27<00:00,  1.31it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.22it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 17: {'accuracy': 0.8799019607843137, 'f1': 0.9129662522202486}\n"
+      "epoch 17: {'accuracy': 0.8627450980392157, 'f1': 0.903448275862069}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [01:18<00:00,  1.47it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.51it/s]\n"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:28<00:00,  1.31it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:04<00:00,  2.65it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "epoch 18: {'accuracy': 0.8799019607843137, 'f1': 0.913884007029877}\n"
+      "epoch 18: {'accuracy': 0.8700980392156863, 'f1': 0.9078260869565218}\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|█████████████████████████████████████████████████████████████| 115/115 [01:26<00:00,  1.32it/s]\n",
-      "100%|███████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.48it/s]"
+      "100%|████████████████████████████████████████████████████████████████████████████████████████| 115/115 [01:27<00:00,  1.32it/s]\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:05<00:00,  2.45it/s]"
      ]
     },
     {
@@ -1133,25 +538,146 @@
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "afaf42dd",
+   "cell_type": "markdown",
+   "id": "7734299c",
    "metadata": {},
-   "outputs": [],
-   "source": []
+   "source": [
+    "## Share adapters on the 🤗 Hub"
+   ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
-   "id": "14504bf5",
+   "execution_count": 8,
+   "id": "afaf42dd",
    "metadata": {},
-   "outputs": [],
-   "source": []
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "CommitInfo(commit_url='https://huggingface.co/smangrul/roberta-large-peft-prefix-tuning/commit/a00e05a4c9a68e700221784f8e073c2e194637c3', commit_message='Upload model', commit_description='', oid='a00e05a4c9a68e700221784f8e073c2e194637c3', pr_url=None, pr_revision=None, pr_num=None)"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model.push_to_hub(\"smangrul/roberta-large-peft-prefix-tuning\", use_auth_token=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "42b20e77",
+   "metadata": {},
+   "source": [
+    "## Load adapters from the Hub\n",
+    "\n",
+    "You can also directly load adapters from the Hub using the commands below:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "868e7580",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "2ce57b4de8ae4f868115733abc2fb883",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading:   0%|          | 0.00/373 [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of the model checkpoint at roberta-large were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.dense.weight', 'roberta.pooler.dense.weight', 'lm_head.bias', 'lm_head.decoder.weight', 'lm_head.dense.bias']\n",
+      "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+      "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
+      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-large and are newly initialized: ['classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias', 'classifier.dense.weight']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ace158c926a44b31a9b0ea80411bd7a9",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Downloading:   0%|          | 0.00/8.14M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "  0%|                                                                                                   | 0/13 [00:00<?, ?it/s]You're using a RobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
+      "100%|██████████████████████████████████████████████████████████████████████████████████████████| 13/13 [00:06<00:00,  2.04it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'accuracy': 0.8774509803921569, 'f1': 0.9125874125874125}\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "from peft import PeftModel, PeftConfig\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
+    "\n",
+    "peft_model_id = \"smangrul/roberta-large-peft-prefix-tuning\"\n",
+    "config = PeftConfig.from_pretrained(peft_model_id)\n",
+    "inference_model = AutoModelForSequenceClassification.from_pretrained(config.base_model_name_or_path)\n",
+    "tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)\n",
+    "\n",
+    "# Load the Lora model\n",
+    "inference_model = PeftModel.from_pretrained(inference_model, peft_model_id)\n",
+    "\n",
+    "inference_model.to(device)\n",
+    "inference_model.eval()\n",
+    "for step, batch in enumerate(tqdm(eval_dataloader)):\n",
+    "    batch.to(device)\n",
+    "    with torch.no_grad():\n",
+    "        outputs = inference_model(**batch)\n",
+    "    predictions = outputs.logits.argmax(dim=-1)\n",
+    "    predictions, references = predictions, batch[\"labels\"]\n",
+    "    metric.add_batch(\n",
+    "        predictions=predictions,\n",
+    "        references=references,\n",
+    "    )\n",
+    "\n",
+    "eval_metric = metric.compute()\n",
+    "print(eval_metric)\n"
+   ]
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py
index 32937ae..ea24090 100644
--- a/src/peft/peft_model.py
+++ b/src/peft/peft_model.py
@@ -94,7 +94,14 @@ class PeftModel(PushToHubMixin, torch.nn.Module):
             raise ValueError(f"Provided path ({save_directory}) should be a directory, not a file")
         os.makedirs(save_directory, exist_ok=True)
 
-        # save the config
+        for param in self.parameters():
+            param.requires_grad = False  # freeze the model
+
+        # save only the trainable weights
+        output_state_dict = get_peft_model_state_dict(self, kwargs.get("state_dict", None))
+        torch.save(output_state_dict, os.path.join(save_directory, WEIGHTS_NAME))
+
+        # save the config and change the inference mode to `True`
         if self.peft_config.base_model_name_or_path is None:
             self.peft_config.base_model_name_or_path = (
                 self.base_model.__dict__.get("name_or_path", None)
@@ -104,13 +111,6 @@ class PeftModel(PushToHubMixin, torch.nn.Module):
         self.peft_config.inference_mode = True
         self.peft_config.save_pretrained(save_directory)
 
-        for param in self.parameters():
-            param.requires_grad = False  # freeze the model
-
-        # save only the trainable weights
-        output_state_dict = get_peft_model_state_dict(self, kwargs.get("state_dict", None))
-        torch.save(output_state_dict, os.path.join(save_directory, WEIGHTS_NAME))
-
     @classmethod
     def from_pretrained(cls, model, model_id, **kwargs):
         r"""