From b07ea17f49fb55cd840a35a15f7b5ef19721f6fe Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Wed, 8 Feb 2023 14:55:08 +0530 Subject: [PATCH] update examples --- ...a_clm_accelerate_big_model_inference.ipynb | 1650 ++--------------- ...ora_seq2seq_accelerate_ds_zero3_offload.py | 54 +- .../peft_lora_seq2seq_accelerate_fsdp.py | 9 +- 3 files changed, 181 insertions(+), 1532 deletions(-) diff --git a/examples/causal_language_modeling/peft_lora_clm_accelerate_big_model_inference.ipynb b/examples/causal_language_modeling/peft_lora_clm_accelerate_big_model_inference.ipynb index 2f2f1ef..d500099 100644 --- a/examples/causal_language_modeling/peft_lora_clm_accelerate_big_model_inference.ipynb +++ b/examples/causal_language_modeling/peft_lora_clm_accelerate_big_model_inference.ipynb @@ -5,10 +5,26 @@ "execution_count": 1, "id": "71fbfca2", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===================================BUG REPORT===================================\n", + "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n", + "For effortless bug reporting copy-paste your error into this form: https://docs.google.com/forms/d/e/1FAIpQLScPB8emS3Thkp66nvqwmjTEgxp8Y9ufuWTzFyr9kJ5AoI47dQ/viewform?usp=sf_link\n", + "================================================================================\n", + "CUDA SETUP: CUDA runtime path found: /home/sourab/miniconda3/envs/ml/lib/libcudart.so\n", + "CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n", + "CUDA SETUP: Detected CUDA version 117\n", + "CUDA SETUP: Loading binary /home/sourab/miniconda3/envs/ml/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n" + ] + } + ], "source": [ "from transformers import AutoModelForCausalLM\n", - "from peft import get_peft_config,get_peft_model, get_peft_model_state_dict, set_peft_model_state_dict, LoraConfig, TaskType, peft_model_load_and_dispatch\n", + "from peft import PeftModel, PeftConfig\n", "import torch\n", "from datasets import load_dataset\n", "import os\n", @@ -21,10 +37,7 @@ "device = \"cuda\"\n", "model_name_or_path = \"bigscience/bloomz-7b1\"\n", "tokenizer_name_or_path = \"bigscience/bloomz-7b1\"\n", - "peft_config = LoraConfig(task_type=TaskType.CAUSAL_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1)\n", - "\n", "dataset_name = \"twitter_complaints\"\n", - "checkpoint_name = \"/home/sourab/\"+f\"{dataset_name}_{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}_v1.pt\".replace(\"/\", \"_\")\n", "text_column = \"Tweet text\"\n", "label_column = \"text_label\"\n", "max_length=64\n", @@ -49,7 +62,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "92d0876af16b4525a124c79cf2da14b2", + "model_id": "d1160ad661de452dbce6944a881a160b", "version_major": 2, "version_minor": 0 }, @@ -64,8 +77,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/ought___raft/twitter_complaints/1.1.0/79c4de1312c1e3730043f7db07179c914f48403101f7124e2fe336f6f54d9f84/cache-05388978db6af01d.arrow\n", - "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/ought___raft/twitter_complaints/1.1.0/79c4de1312c1e3730043f7db07179c914f48403101f7124e2fe336f6f54d9f84/cache-e3fade69c4ae889a.arrow\n" + "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/ought___raft/twitter_complaints/1.1.0/79c4de1312c1e3730043f7db07179c914f48403101f7124e2fe336f6f54d9f84/cache-20a7622c86d80cdf.arrow\n", + "Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/ought___raft/twitter_complaints/1.1.0/79c4de1312c1e3730043f7db07179c914f48403101f7124e2fe336f6f54d9f84/cache-5f1431311da05803.arrow\n" ] }, { @@ -132,7 +145,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "8463b60edfa8461da430caf24083ed4f", + "model_id": "10cabeec92ab428f9a660ebaecbaf865", "version_major": 2, "version_minor": 0 }, @@ -146,7 +159,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "5451d884213749e7941a722cb911ea6b", + "model_id": "8a344e989ab34c71b230acee68b477e8", "version_major": 2, "version_minor": 0 }, @@ -223,7 +236,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "85e2d59da7c447af957fc43b696d2e9e", + "model_id": "293c2191c9674123afc58cfa22c0cf57", "version_major": 2, "version_minor": 0 }, @@ -237,7 +250,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "27fc92fedbc7458585b6029dd9c06310", + "model_id": "c376f3397f574c9084946ab171f43c4f", "version_major": 2, "version_minor": 0 }, @@ -464,199 +477,6 @@ { "cell_type": "code", "execution_count": 5, - "id": "accc5012", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'input_ids': tensor([[ 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 227985, 5484, 915, 2566,\n", - " 31934, 5227, 6640, 16261, 87843, 17, 130462, 9600, 169668,\n", - " 28, 13604, 112581, 40, 77658, 915, 210, 1936, 106863,\n", - " 3],\n", - " [ 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 227985, 5484, 915, 2566, 49, 656, 266,\n", - " 53312, 44639, 4936, 17, 77658, 915, 210, 1936, 106863,\n", - " 3],\n", - " [ 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 227985, 5484, 915, 2566,\n", - " 57647, 327, 38804, 86, 35631, 368, 7733, 4676, 427,\n", - " 10665, 57903, 664, 267, 6917, 18706, 427, 368, 16698,\n", - " 35633, 3383, 27409, 34, 77658, 915, 210, 1936, 106863,\n", - " 3],\n", - " [ 3, 3, 3, 3, 3, 3, 3, 227985, 5484,\n", - " 915, 419, 15096, 510, 67508, 67508, 75164, 9293, 2566,\n", - " 53, 34867, 2566, 15225, 10885, 7800, 252, 17, 1931,\n", - " 279, 12161, 10082, 2335, 18704, 361, 56006, 473, 17760,\n", - " 6582, 4472, 718, 3866, 1804, 44783, 56006, 4570, 94678,\n", - " 4346, 87843, 17, 1594, 14663, 74, 59, 92, 28,\n", - " 91, 5871, 27, 71, 77658, 915, 210, 1936, 106863,\n", - " 3],\n", - " [ 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 227985, 5484, 915, 43504, 51648, 5456,\n", - " 176088, 34, 57180, 28627, 269, 13041, 2566, 114242, 672,\n", - " 2338, 56114, 427, 4054, 530, 182640, 7963, 427, 19134,\n", - " 718, 4, 2550, 55061, 17209, 2550, 114242, 672, 9702,\n", - " 90, 647, 4346, 87843, 17, 130462, 40081, 10881, 73,\n", - " 84, 38, 624, 43, 77658, 915, 210, 1936, 106863,\n", - " 3],\n", - " [ 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 227985, 5484, 915, 2566, 58, 24673, 21,\n", - " 34274, 1244, 613, 2910, 77658, 915, 210, 1936, 106863,\n", - " 3],\n", - " [ 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 227985,\n", - " 5484, 915, 2566, 182441, 55, 35040, 435, 16796, 79920,\n", - " 427, 661, 6355, 17, 5568, 2213, 3172, 960, 126355,\n", - " 6216, 5559, 61273, 23181, 77658, 915, 210, 1936, 106863,\n", - " 3],\n", - " [ 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 3, 3,\n", - " 3, 3, 3, 3, 3, 3, 3, 227985, 5484,\n", - " 915, 473, 3370, 29408, 973, 44805, 427, 162074, 72,\n", - " 919, 2566, 951, 23323, 228277, 351, 613, 368, 10087,\n", - " 9440, 473, 15017, 4, 77658, 915, 210, 1936, 106863,\n", - " 3]]),\n", - " 'attention_mask': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", - " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n", - " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", - " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n", - " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", - " 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n", - " [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n", - " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n", - " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", - " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n", - " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", - " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n", - " [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", - " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]),\n", - " 'labels': tensor([[ -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, 1936, 106863,\n", - " 3],\n", - " [ -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, 1936, 106863,\n", - " 3],\n", - " [ -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, 1936, 106863,\n", - " 3],\n", - " [ -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, 1936, 106863,\n", - " 3],\n", - " [ -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, 1936, 106863,\n", - " 3],\n", - " [ -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, 1936, 106863,\n", - " 3],\n", - " [ -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, 1936, 106863,\n", - " 3],\n", - " [ -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, -100, -100,\n", - " -100, -100, -100, -100, -100, -100, -100, 1936, 106863,\n", - " 3]])}" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "next(iter(train_dataloader))" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "218df807", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "425" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "len(test_dataloader)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, "id": "9caac014", "metadata": {}, "outputs": [ @@ -664,1265 +484,91 @@ "name": "stderr", "output_type": "stream", "text": [ - "/home/sourab/pet/src/pet/tuners/lora.py:106: UserWarning: fan_in_fan_out is set to True but the target module is not a Conv1D. Setting fan_in_fan_out to False.\n", + "/home/sourab/pet/src/peft/tuners/lora.py:143: UserWarning: fan_in_fan_out is set to True but the target module is not a Conv1D. Setting fan_in_fan_out to False.\n", " warnings.warn(\n" ] }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trainable params: 3932160 || all params: 7072948224 || trainable%: 0.055594355783029126\n" - ] - }, { "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "bc38030106a14173a1363eb1ee388eda", + "version_major": 2, + "version_minor": 0 + }, "text/plain": [ - "PETModelForCausalLM(\n", - " (base_model): LoRAModel(\n", - " (model): BloomForCausalLM(\n", - " (transformer): BloomModel(\n", - " (word_embeddings): Embedding(250880, 4096)\n", - " (word_embeddings_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (h): ModuleList(\n", - " (0): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (1): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (2): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (3): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (4): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (5): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (6): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (7): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (8): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (9): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (10): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (11): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (12): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (13): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (14): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (15): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (16): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (17): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (18): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (19): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (20): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (21): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (22): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (23): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (24): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (25): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (26): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (27): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (28): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " (29): BloomBlock(\n", - " (input_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (self_attention): BloomAttention(\n", - " (query_key_value): MergedLinear(\n", - " in_features=4096, out_features=12288, bias=True\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=4096, out_features=16, bias=False)\n", - " (lora_B): Conv1d(16, 8192, kernel_size=(1,), stride=(1,), groups=2, bias=False)\n", - " )\n", - " (dense): Linear(in_features=4096, out_features=4096, bias=True)\n", - " (attention_dropout): Dropout(p=0.0, inplace=False)\n", - " )\n", - " (post_attention_layernorm): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " (mlp): BloomMLP(\n", - " (dense_h_to_4h): Linear(in_features=4096, out_features=16384, bias=True)\n", - " (gelu_impl): BloomGelu()\n", - " (dense_4h_to_h): Linear(in_features=16384, out_features=4096, bias=True)\n", - " )\n", - " )\n", - " )\n", - " (ln_f): LayerNorm((4096,), eps=1e-05, elementwise_affine=True)\n", - " )\n", - " (lm_head): Linear(in_features=4096, out_features=250880, bias=False)\n", - " )\n", - " )\n", - ")" + "Downloading: 0%| | 0.00/15.8M [00:00