{ "cells": [ { "cell_type": "code", "execution_count": 17, "id": "5f93b7d1", "metadata": {}, "outputs": [], "source": [ "from transformers import AutoModelForSeq2SeqLM\n", "from pet import get_pet_config,get_pet_model, get_pet_model_state_dict\n", "import torch\n", "from datasets import load_dataset\n", "import os\n", "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", "from transformers import AutoTokenizer\n", "from torch.utils.data import DataLoader\n", "from transformers import default_data_collator,get_linear_schedule_with_warmup\n", "from tqdm import tqdm\n", "from datasets import load_dataset\n", "\n", "device = \"cuda\"\n", "model_name_or_path = \"bigscience/mt0-large\"\n", "tokenizer_name_or_path = \"bigscience/mt0-large\"\n", "\n", "config = {\n", " \"pet_type\":\"LORA\",\n", " \"task_type\":\"SEQ_2_SEQ_LM\",\n", " \"r\":16,\n", " \"lora_alpha\": 32,\n", " \"lora_dropout\": 0.1\n", "}\n", "checkpoint_name = \"financial_sentiment_analysis_lora_v1.pt\"\n", "text_column = \"sentence\"\n", "label_column = \"text_label\"\n", "max_length=128\n", "lr = 1e-3\n", "num_epochs = 3\n", "batch_size=8\n" ] }, { "cell_type": "code", "execution_count": 2, "id": "8d0850ac", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "trainable params: 4718592 || all params: 1234299904 || trainable%: 0.38228893842642636\n" ] }, { "data": { "text/plain": [ "PETModelForSeq2SeqLM(\n", " (base_model): LoRAModel(\n", " (model): MT5ForConditionalGeneration(\n", " (shared): Embedding(250112, 1024)\n", " (encoder): T5Stack(\n", " (embed_tokens): Embedding(250112, 1024)\n", " (block): ModuleList(\n", " (0): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " (relative_attention_bias): Embedding(32, 16)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (1): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (2): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (3): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (4): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (5): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (6): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (7): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (8): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (9): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (10): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (11): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (12): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (13): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (14): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (15): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (16): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (17): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (18): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (19): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (20): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (21): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (22): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (23): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " )\n", " (final_layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (decoder): T5Stack(\n", " (embed_tokens): Embedding(250112, 1024)\n", " (block): ModuleList(\n", " (0): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " (relative_attention_bias): Embedding(32, 16)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (1): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (2): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (3): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (4): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (5): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (6): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (7): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (8): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (9): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (10): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (11): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (12): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (13): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (14): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (15): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (16): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (17): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (18): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (19): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (20): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (21): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (22): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " (23): T5Block(\n", " (layer): ModuleList(\n", " (0): T5LayerSelfAttention(\n", " (SelfAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (1): T5LayerCrossAttention(\n", " (EncDecAttention): T5Attention(\n", " (q): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", " (v): Linear(\n", " in_features=1024, out_features=1024, bias=False\n", " (lora_dropout): Dropout(p=0.1, inplace=False)\n", " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", " )\n", " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (2): T5LayerFF(\n", " (DenseReluDense): T5DenseGatedActDense(\n", " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " (act): NewGELUActivation()\n", " )\n", " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " )\n", " )\n", " )\n", " (final_layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", " (dropout): Dropout(p=0.1, inplace=False)\n", " )\n", " (lm_head): Linear(in_features=1024, out_features=250112, bias=False)\n", " )\n", " )\n", ")" ] }, "execution_count": 2, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# creating model\n", "pet_config = get_pet_config(config)\n", "\n", "model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path)\n", "model = get_pet_model(model, pet_config)\n", "model.print_trainable_parameters()\n", "model" ] }, { "cell_type": "code", "execution_count": 3, "id": "4ee2babf", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/sourab/miniconda3/envs/ml/lib/python3.10/site-packages/huggingface_hub/utils/_deprecation.py:97: FutureWarning: Deprecated argument(s) used in 'dataset_info': token. Will not be supported from version '0.12'.\n", " warnings.warn(message, FutureWarning)\n", "Found cached dataset financial_phrasebank (/home/sourab/.cache/huggingface/datasets/financial_phrasebank/sentences_allagree/1.0.0/550bde12e6c30e2674da973a55f57edde5181d53f5a5a34c1531c53f93b7e141)\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "6de075f8208349108291ac5ab7f5c980", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/1 [00:00