peft/examples/pet_lora_seq2seq.ipynb

In [17]:

from transformers import AutoModelForSeq2SeqLM
from pet import get_pet_config,get_pet_model, get_pet_model_state_dict
import torch
from datasets import load_dataset
import os
os.environ["TOKENIZERS_PARALLELISM"] = "false"
from transformers import AutoTokenizer
from torch.utils.data import DataLoader
from transformers import default_data_collator,get_linear_schedule_with_warmup
from tqdm import tqdm
from datasets import load_dataset

device = "cuda"
model_name_or_path = "bigscience/mt0-large"
tokenizer_name_or_path = "bigscience/mt0-large"

config = {
    "pet_type":"LORA",
    "task_type":"SEQ_2_SEQ_LM",
    "r":16,
    "lora_alpha": 32,
    "lora_dropout": 0.1
}
checkpoint_name = "financial_sentiment_analysis_lora_v1.pt"
text_column = "sentence"
label_column = "text_label"
max_length=128
lr = 1e-3
num_epochs = 3
batch_size=8

In [2]:

# creating model
pet_config = get_pet_config(config)

model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path)
model = get_pet_model(model, pet_config)
model.print_trainable_parameters()
model

Out [2]:

trainable params: 4718592 || all params: 1234299904 || trainable%: 0.38228893842642636

PETModelForSeq2SeqLM(
  (base_model): LoRAModel(
    (model): MT5ForConditionalGeneration(
      (shared): Embedding(250112, 1024)
      (encoder): T5Stack(
        (embed_tokens): Embedding(250112, 1024)
        (block): ModuleList(
          (0): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                  (relative_attention_bias): Embedding(32, 16)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (1): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (2): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (3): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (4): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (5): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (6): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (7): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (8): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (9): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (10): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (11): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (12): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (13): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (14): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (15): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (16): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (17): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (18): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (19): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (20): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (21): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (22): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (23): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
        )
        (final_layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (decoder): T5Stack(
        (embed_tokens): Embedding(250112, 1024)
        (block): ModuleList(
          (0): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                  (relative_attention_bias): Embedding(32, 16)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (1): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (2): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (3): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (4): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (5): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (6): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (7): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (8): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (9): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (10): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (11): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (12): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (13): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (14): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (15): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (16): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (17): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (18): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (19): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (20): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (21): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (22): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
          (23): T5Block(
            (layer): ModuleList(
              (0): T5LayerSelfAttention(
                (SelfAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (1): T5LayerCrossAttention(
                (EncDecAttention): T5Attention(
                  (q): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (k): Linear(in_features=1024, out_features=1024, bias=False)
                  (v): Linear(
                    in_features=1024, out_features=1024, bias=False
                    (lora_dropout): Dropout(p=0.1, inplace=False)
                    (lora_A): Linear(in_features=1024, out_features=16, bias=False)
                    (lora_B): Linear(in_features=16, out_features=1024, bias=False)
                  )
                  (o): Linear(in_features=1024, out_features=1024, bias=False)
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
              (2): T5LayerFF(
                (DenseReluDense): T5DenseGatedActDense(
                  (wi_0): Linear(in_features=1024, out_features=2816, bias=False)
                  (wi_1): Linear(in_features=1024, out_features=2816, bias=False)
                  (wo): Linear(in_features=2816, out_features=1024, bias=False)
                  (dropout): Dropout(p=0.1, inplace=False)
                  (act): NewGELUActivation()
                )
                (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
                (dropout): Dropout(p=0.1, inplace=False)
              )
            )
          )
        )
        (final_layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (lm_head): Linear(in_features=1024, out_features=250112, bias=False)
    )
  )
)

In [3]:

# loading dataset
dataset = load_dataset("financial_phrasebank", 'sentences_allagree')
dataset = dataset["train"].train_test_split(test_size=0.1)
dataset["validation"] = dataset["test"]
del(dataset["test"])

classes = dataset["train"].features["label"].names
dataset = dataset.map(
    lambda x: {"text_label": [classes[label] for label in x["label"]]},
    batched=True,
    num_proc=1,
    
)

dataset["train"][0]

Out [3]:

/home/sourab/miniconda3/envs/ml/lib/python3.10/site-packages/huggingface_hub/utils/_deprecation.py:97: FutureWarning: Deprecated argument(s) used in 'dataset_info': token. Will not be supported from version '0.12'.
  warnings.warn(message, FutureWarning)
Found cached dataset financial_phrasebank (/home/sourab/.cache/huggingface/datasets/financial_phrasebank/sentences_allagree/1.0.0/550bde12e6c30e2674da973a55f57edde5181d53f5a5a34c1531c53f93b7e141)

  0%|          | 0/1 [00:00<?, ?it/s]

  0%|          | 0/3 [00:00<?, ?ba/s]

  0%|          | 0/1 [00:00<?, ?ba/s]

{'sentence': 'The order was valued at USD12 .2 m.',
 'label': 1,
 'text_label': 'neutral'}

In [4]:

# data preprocessing
tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
def preprocess_function(examples):
    inputs = examples[text_column]
    targets = examples[label_column]
    model_inputs = tokenizer(inputs, max_length=max_length, padding="max_length", truncation=True, return_tensors="pt")
    labels = tokenizer(targets, max_length=3, padding="max_length", truncation=True, return_tensors="pt")
    labels = labels["input_ids"]
    labels[labels==tokenizer.pad_token_id] = -100
    model_inputs["labels"] = labels
    return model_inputs

processed_datasets = dataset.map(
            preprocess_function,
            batched=True,
            num_proc=1,
            remove_columns=dataset["train"].column_names,
            load_from_cache_file=False,
            desc="Running tokenizer on dataset",
        )

train_dataset = processed_datasets["train"]
eval_dataset = processed_datasets["validation"]

train_dataloader = DataLoader(
        train_dataset, shuffle=True, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True
    )
eval_dataloader = DataLoader(eval_dataset, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True)



    

Running tokenizer on dataset:   0%|          | 0/3 [00:00<?, ?ba/s]

Running tokenizer on dataset:   0%|          | 0/1 [00:00<?, ?ba/s]

In [5]:

# optimizer and lr scheduler
optimizer = torch.optim.AdamW(model.parameters(), lr=lr)
lr_scheduler = get_linear_schedule_with_warmup(
    optimizer=optimizer,
    num_warmup_steps=0,
    num_training_steps=(len(train_dataloader) * num_epochs),
)

In [6]:

# training and evaluation
model = model.to(device)

for epoch in range(num_epochs):
    model.train()
    total_loss = 0
    for step, batch in enumerate(tqdm(train_dataloader)):
        batch = {k: v.to(device) for k, v in batch.items()}
        outputs = model(**batch)
        loss = outputs.loss
        total_loss += loss.detach().float()
        loss.backward()
        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()

    model.eval()
    eval_loss = 0
    eval_preds = []
    for step, batch in enumerate(tqdm(eval_dataloader)):
        batch = {k: v.to(device) for k, v in batch.items()}
        with torch.no_grad():
            outputs = model(**batch)
        loss = outputs.loss
        eval_loss += loss.detach().float()
        eval_preds.extend(tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True))

    eval_epoch_loss = eval_loss/len(train_dataloader)
    eval_ppl = torch.exp(eval_epoch_loss)
    train_epoch_loss = total_loss/len(eval_dataloader)
    train_ppl = torch.exp(train_epoch_loss)
    print(f"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}")

100%|█████████████████████████████████████████████████████████████| 255/255 [00:53<00:00,  4.80it/s]
100%|███████████████████████████████████████████████████████████████| 29/29 [00:02<00:00, 14.16it/s]

epoch=0: train_ppl=tensor(13.6966, device='cuda:0') train_epoch_loss=tensor(2.6171, device='cuda:0') eval_ppl=tensor(1.0046, device='cuda:0') eval_epoch_loss=tensor(0.0046, device='cuda:0')

100%|█████████████████████████████████████████████████████████████| 255/255 [00:52<00:00,  4.88it/s]
100%|███████████████████████████████████████████████████████████████| 29/29 [00:02<00:00, 14.20it/s]

epoch=1: train_ppl=tensor(1.5893, device='cuda:0') train_epoch_loss=tensor(0.4633, device='cuda:0') eval_ppl=tensor(1.0020, device='cuda:0') eval_epoch_loss=tensor(0.0020, device='cuda:0')

100%|█████████████████████████████████████████████████████████████| 255/255 [00:52<00:00,  4.87it/s]
100%|███████████████████████████████████████████████████████████████| 29/29 [00:02<00:00, 14.18it/s]

epoch=2: train_ppl=tensor(1.3210, device='cuda:0') train_epoch_loss=tensor(0.2784, device='cuda:0') eval_ppl=tensor(1.0026, device='cuda:0') eval_epoch_loss=tensor(0.0026, device='cuda:0')

In [20]:

# print accuracy
correct =0
total = 0
for pred,true in zip(eval_preds, dataset["validation"]["text_label"]):
    if pred.strip()==true.strip():
        correct+=1
    total+=1  
accuracy = correct/total*100
print(f"{accuracy=} % on the evaluation dataset")
print(f"{eval_preds[:10]=}")
print(f"{dataset['validation']['text_label'][:10]=}")

accuracy=98.23788546255507 % on the evaluation dataset
eval_preds[:10]=['neutral', 'neutral', 'positive', 'positive', 'neutral', 'neutral', 'neutral', 'neutral', 'neutral', 'neutral']
dataset['validation']['text_label'][:10]=['neutral', 'neutral', 'positive', 'positive', 'neutral', 'neutral', 'neutral', 'neutral', 'neutral', 'neutral']

In [8]:

# saving model
state_dict = get_pet_model_state_dict(model)
torch.save(state_dict, checkpoint_name)
print(state_dict)

{'base_model.model.encoder.block.0.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0011,  0.0126, -0.0392,  ...,  0.0117, -0.0080,  0.0227],
        [ 0.0239, -0.0152,  0.0174,  ..., -0.0209, -0.0093, -0.0250],
        [ 0.0197,  0.0023,  0.0045,  ..., -0.0073, -0.0106, -0.0038],
        ...,
        [-0.0116,  0.0015,  0.0146,  ...,  0.0017, -0.0237, -0.0077],
        [ 0.0143,  0.0330,  0.0059,  ...,  0.0124,  0.0045, -0.0180],
        [ 0.0095,  0.0024,  0.0352,  ...,  0.0064,  0.0230, -0.0452]],
       device='cuda:0'), 'base_model.model.encoder.block.0.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0161, -0.0037,  0.0014,  ...,  0.0039,  0.0235, -0.0148],
        [-0.0086, -0.0180,  0.0253,  ...,  0.0277,  0.0029, -0.0193],
        [ 0.0004,  0.0172,  0.0078,  ...,  0.0094, -0.0130,  0.0012],
        ...,
        [ 0.0010,  0.0024,  0.0202,  ...,  0.0259, -0.0117,  0.0107],
        [ 0.0256,  0.0210, -0.0293,  ..., -0.0210,  0.0059, -0.0250],
        [-0.0056, -0.0048,  0.0187,  ...,  0.0086, -0.0178,  0.0021]],
       device='cuda:0'), 'base_model.model.encoder.block.0.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0205, -0.0127,  0.0213,  ..., -0.0042,  0.0034,  0.0243],
        [ 0.0197, -0.0426, -0.0326,  ...,  0.0069, -0.0203,  0.0358],
        [-0.0216,  0.0100, -0.0084,  ...,  0.0446, -0.0540,  0.0452],
        ...,
        [ 0.0578, -0.0072,  0.0047,  ..., -0.0078, -0.0231,  0.0116],
        [ 0.0131,  0.0072,  0.0372,  ..., -0.0183, -0.0214, -0.0139],
        [ 0.0013,  0.0402, -0.0062,  ...,  0.0365,  0.0286, -0.0329]],
       device='cuda:0'), 'base_model.model.encoder.block.0.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0046, -0.0044,  0.0061,  ..., -0.0013, -0.0303, -0.0034],
        [ 0.0060,  0.0068, -0.0019,  ..., -0.0046, -0.0336, -0.0189],
        [-0.0039, -0.0053,  0.0089,  ..., -0.0128,  0.0059, -0.0216],
        ...,
        [ 0.0052, -0.0199, -0.0261,  ..., -0.0253,  0.0173, -0.0027],
        [ 0.0098, -0.0081, -0.0105,  ..., -0.0074, -0.0016, -0.0026],
        [ 0.0134, -0.0060,  0.0122,  ...,  0.0021, -0.0014,  0.0148]],
       device='cuda:0'), 'base_model.model.encoder.block.1.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0306, -0.0243, -0.0414,  ...,  0.0121,  0.0280,  0.0006],
        [-0.0425, -0.0292,  0.0076,  ...,  0.0322, -0.0244,  0.0346],
        [-0.0081,  0.0155, -0.0289,  ..., -0.0078,  0.0121, -0.0135],
        ...,
        [ 0.0161,  0.0303, -0.0498,  ..., -0.0148,  0.0022, -0.0115],
        [ 0.0205,  0.0197, -0.0203,  ..., -0.0424, -0.0252, -0.0418],
        [ 0.0098, -0.0085, -0.0391,  ...,  0.0195, -0.0180, -0.0565]],
       device='cuda:0'), 'base_model.model.encoder.block.1.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 1.3011e-02, -8.4894e-03,  1.4550e-02,  ..., -2.8281e-02,
         -1.5803e-02, -6.7014e-03],
        [-6.9723e-03,  2.5044e-02,  1.0645e-03,  ..., -1.9179e-02,
         -2.0764e-02, -1.5413e-02],
        [ 1.1783e-02, -1.5737e-02,  1.4881e-02,  ..., -3.9572e-03,
          1.2995e-02,  2.8500e-02],
        ...,
        [-1.0119e-02,  3.6680e-03,  2.4997e-03,  ..., -1.1717e-02,
          6.6043e-03, -3.2334e-03],
        [-1.1863e-02, -6.9381e-04, -1.1733e-02,  ..., -3.1885e-03,
         -1.4473e-02,  1.3692e-03],
        [-7.2537e-03,  2.4926e-02,  2.9916e-03,  ..., -6.9848e-04,
         -5.9038e-03, -5.7989e-05]], device='cuda:0'), 'base_model.model.encoder.block.1.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0199, -0.0045,  0.0448,  ..., -0.0153, -0.0625, -0.0284],
        [-0.0109, -0.0260, -0.0110,  ...,  0.0036,  0.0067, -0.0112],
        [ 0.0537,  0.0035, -0.0319,  ..., -0.0041, -0.0150,  0.0522],
        ...,
        [ 0.0215,  0.0431, -0.0165,  ...,  0.0086,  0.0072,  0.0050],
        [ 0.0327, -0.0188,  0.0122,  ...,  0.0043, -0.0070, -0.0060],
        [ 0.0123, -0.0429,  0.0311,  ..., -0.0028,  0.0158, -0.0010]],
       device='cuda:0'), 'base_model.model.encoder.block.1.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0024, -0.0146, -0.0314,  ...,  0.0111, -0.0105, -0.0042],
        [ 0.0165,  0.0012, -0.0041,  ...,  0.0039, -0.0018,  0.0084],
        [-0.0278,  0.0229, -0.0050,  ..., -0.0019, -0.0090,  0.0025],
        ...,
        [ 0.0228,  0.0125, -0.0167,  ..., -0.0163, -0.0046,  0.0134],
        [-0.0191, -0.0080,  0.0129,  ...,  0.0015, -0.0096, -0.0112],
        [ 0.0306, -0.0115, -0.0116,  ...,  0.0047, -0.0097, -0.0013]],
       device='cuda:0'), 'base_model.model.encoder.block.2.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0183, -0.0045, -0.0373,  ...,  0.0028, -0.0018,  0.0099],
        [-0.0376,  0.0284, -0.0107,  ..., -0.0196, -0.0107,  0.0114],
        [ 0.0004,  0.0118, -0.0360,  ..., -0.0594,  0.0541,  0.0094],
        ...,
        [ 0.0039, -0.0022,  0.0018,  ...,  0.0467,  0.0007, -0.0348],
        [-0.0036,  0.0106,  0.0272,  ...,  0.0413,  0.0270,  0.0300],
        [-0.0325, -0.0090,  0.0226,  ..., -0.0035,  0.0270, -0.0215]],
       device='cuda:0'), 'base_model.model.encoder.block.2.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0052,  0.0233, -0.0136,  ..., -0.0086,  0.0216,  0.0165],
        [ 0.0174,  0.0217,  0.0273,  ..., -0.0157,  0.0139,  0.0167],
        [ 0.0038, -0.0342,  0.0208,  ...,  0.0044, -0.0041,  0.0206],
        ...,
        [-0.0028,  0.0279,  0.0068,  ..., -0.0180,  0.0110,  0.0245],
        [-0.0083,  0.0002,  0.0032,  ...,  0.0041,  0.0075,  0.0023],
        [ 0.0368, -0.0012, -0.0103,  ..., -0.0077, -0.0096, -0.0221]],
       device='cuda:0'), 'base_model.model.encoder.block.2.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0620, -0.0185, -0.0250,  ..., -0.0078,  0.0048,  0.0218],
        [ 0.0750, -0.0434,  0.0007,  ..., -0.0083,  0.0214,  0.0416],
        [ 0.0181, -0.0213,  0.0101,  ...,  0.0022,  0.0036,  0.0140],
        ...,
        [-0.0114,  0.0142, -0.0239,  ..., -0.0055,  0.0138,  0.0047],
        [-0.0246, -0.0030,  0.0016,  ...,  0.0156, -0.0082, -0.0074],
        [-0.0391,  0.0287, -0.0008,  ..., -0.0037, -0.0228, -0.0337]],
       device='cuda:0'), 'base_model.model.encoder.block.2.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0043,  0.0104,  0.0072,  ..., -0.0040,  0.0189, -0.0040],
        [ 0.0302,  0.0278,  0.0314,  ..., -0.0275,  0.0137, -0.0292],
        [-0.0450, -0.0463, -0.0416,  ...,  0.0435, -0.0480,  0.0439],
        ...,
        [-0.0043, -0.0020,  0.0152,  ..., -0.0201,  0.0059, -0.0025],
        [-0.0169, -0.0239, -0.0168,  ...,  0.0162,  0.0110,  0.0223],
        [ 0.0274,  0.0228,  0.0166,  ..., -0.0190, -0.0184, -0.0188]],
       device='cuda:0'), 'base_model.model.encoder.block.3.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0117, -0.0470, -0.0126,  ...,  0.0142, -0.0365, -0.0033],
        [ 0.0198,  0.0106,  0.0006,  ..., -0.0219,  0.0049,  0.0288],
        [ 0.0024,  0.0177,  0.0296,  ..., -0.0142,  0.0018, -0.0186],
        ...,
        [-0.0298, -0.0051,  0.0024,  ...,  0.0243, -0.0158,  0.0213],
        [-0.0088, -0.0265,  0.0339,  ..., -0.0116, -0.0214, -0.0225],
        [-0.0259, -0.0074,  0.0242,  ...,  0.0003, -0.0149,  0.0445]],
       device='cuda:0'), 'base_model.model.encoder.block.3.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0196,  0.0119, -0.0112,  ...,  0.0255, -0.0357, -0.0064],
        [-0.0150,  0.0146,  0.0108,  ..., -0.0005, -0.0003,  0.0082],
        [-0.0081, -0.0125,  0.0033,  ..., -0.0002,  0.0076, -0.0156],
        ...,
        [-0.0096,  0.0218, -0.0031,  ...,  0.0057, -0.0008,  0.0058],
        [ 0.0084, -0.0011,  0.0069,  ..., -0.0263,  0.0075, -0.0034],
        [ 0.0040,  0.0189,  0.0171,  ..., -0.0186,  0.0084,  0.0118]],
       device='cuda:0'), 'base_model.model.encoder.block.3.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0178,  0.0288,  0.0192,  ...,  0.0078, -0.0146, -0.0158],
        [-0.0173,  0.0058, -0.0253,  ..., -0.0172, -0.0134, -0.0071],
        [-0.0167, -0.0333, -0.0169,  ..., -0.0249, -0.0090,  0.0195],
        ...,
        [ 0.0093, -0.0083,  0.0031,  ...,  0.0143,  0.0326,  0.0302],
        [-0.0282,  0.0131,  0.0045,  ..., -0.0152,  0.0099, -0.0038],
        [-0.0027, -0.0236, -0.0395,  ..., -0.0288,  0.0060, -0.0122]],
       device='cuda:0'), 'base_model.model.encoder.block.3.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0041,  0.0012,  0.0171,  ...,  0.0103, -0.0087, -0.0037],
        [ 0.0061, -0.0059,  0.0004,  ...,  0.0228, -0.0394,  0.0151],
        [-0.0123, -0.0029,  0.0168,  ...,  0.0022, -0.0009,  0.0042],
        ...,
        [-0.0318,  0.0018,  0.0162,  ..., -0.0191,  0.0094,  0.0015],
        [-0.0176,  0.0092,  0.0165,  ..., -0.0086,  0.0150,  0.0036],
        [ 0.0359,  0.0136, -0.0198,  ...,  0.0134, -0.0206, -0.0085]],
       device='cuda:0'), 'base_model.model.encoder.block.4.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0325,  0.0371,  0.0161,  ..., -0.0039, -0.0107, -0.0066],
        [-0.0440, -0.0042,  0.0123,  ..., -0.0205, -0.0390,  0.0127],
        [ 0.0115, -0.0246, -0.0256,  ..., -0.0113,  0.0181, -0.0191],
        ...,
        [-0.0033,  0.0258,  0.0214,  ...,  0.0249, -0.0544,  0.0062],
        [-0.0294, -0.0302, -0.0316,  ..., -0.0032, -0.0195,  0.0247],
        [-0.0197, -0.0115,  0.0108,  ...,  0.0308, -0.0469, -0.0307]],
       device='cuda:0'), 'base_model.model.encoder.block.4.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0078, -0.0045,  0.0053,  ...,  0.0212,  0.0162, -0.0151],
        [-0.0228,  0.0114,  0.0248,  ...,  0.0080,  0.0110, -0.0130],
        [-0.0307,  0.0002,  0.0129,  ..., -0.0120, -0.0139, -0.0271],
        ...,
        [-0.0056, -0.0221,  0.0047,  ...,  0.0003,  0.0098, -0.0024],
        [ 0.0210,  0.0042,  0.0005,  ...,  0.0077, -0.0075,  0.0073],
        [ 0.0255,  0.0454,  0.0152,  ...,  0.0398,  0.0391, -0.0117]],
       device='cuda:0'), 'base_model.model.encoder.block.4.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0198, -0.0339,  0.0026,  ...,  0.0093,  0.0370,  0.0019],
        [-0.0242, -0.0109,  0.0368,  ..., -0.0427, -0.0158, -0.0527],
        [-0.0196,  0.0233,  0.0298,  ...,  0.0180, -0.0326,  0.0129],
        ...,
        [ 0.0229,  0.0284, -0.0155,  ...,  0.0192,  0.0025,  0.0028],
        [ 0.0174, -0.0308,  0.0193,  ...,  0.0195, -0.0141,  0.0278],
        [ 0.0147,  0.0028, -0.0325,  ..., -0.0281,  0.0010,  0.0087]],
       device='cuda:0'), 'base_model.model.encoder.block.4.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0090,  0.0005, -0.0142,  ..., -0.0017, -0.0026, -0.0015],
        [ 0.0117, -0.0037,  0.0098,  ..., -0.0013,  0.0009, -0.0046],
        [-0.0163, -0.0004, -0.0039,  ...,  0.0007,  0.0109,  0.0066],
        ...,
        [-0.0187,  0.0198,  0.0076,  ...,  0.0136,  0.0249, -0.0021],
        [ 0.0008,  0.0045, -0.0036,  ..., -0.0057,  0.0173,  0.0004],
        [-0.0046, -0.0078, -0.0104,  ..., -0.0017, -0.0123,  0.0134]],
       device='cuda:0'), 'base_model.model.encoder.block.5.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 4.6854e-02,  2.2295e-02,  6.4651e-03,  ...,  2.5293e-02,
          2.1630e-02,  1.2340e-02],
        [ 1.3335e-02,  3.8377e-02, -4.0418e-02,  ..., -1.1633e-02,
         -1.8374e-02, -1.1622e-02],
        [-5.3741e-02,  7.6245e-03,  5.9155e-02,  ..., -1.7577e-02,
         -4.4678e-02, -1.1400e-02],
        ...,
        [-3.7519e-02,  7.7807e-03,  2.1634e-02,  ..., -1.2836e-02,
         -3.3665e-02, -3.3249e-05],
        [ 8.0464e-03, -1.4498e-03,  1.2061e-02,  ...,  2.6527e-02,
          2.2661e-03,  1.5226e-02],
        [ 1.3127e-02,  3.4243e-02, -7.6076e-03,  ..., -1.7030e-02,
          3.7331e-02, -2.0033e-02]], device='cuda:0'), 'base_model.model.encoder.block.5.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0211,  0.0119,  0.0196,  ...,  0.0310,  0.0261,  0.0212],
        [ 0.0150,  0.0073, -0.0072,  ..., -0.0296, -0.0019, -0.0055],
        [ 0.0040, -0.0227, -0.0023,  ..., -0.0025, -0.0282, -0.0286],
        ...,
        [-0.0339,  0.0068,  0.0066,  ...,  0.0262,  0.0112,  0.0062],
        [ 0.0111,  0.0063, -0.0170,  ..., -0.0230, -0.0087,  0.0053],
        [-0.0182,  0.0041,  0.0143,  ...,  0.0118,  0.0134, -0.0034]],
       device='cuda:0'), 'base_model.model.encoder.block.5.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0077, -0.0111,  0.0294,  ..., -0.0395,  0.0136,  0.0262],
        [-0.0040, -0.0357, -0.0582,  ..., -0.0002, -0.0220, -0.0344],
        [ 0.0358,  0.0259,  0.0419,  ...,  0.0121,  0.0241, -0.0243],
        ...,
        [ 0.0032, -0.0115, -0.0457,  ...,  0.0290,  0.0213, -0.0060],
        [ 0.0011,  0.0236,  0.0375,  ...,  0.0009,  0.0075,  0.0216],
        [-0.0085,  0.0274,  0.0160,  ...,  0.0069, -0.0044,  0.0034]],
       device='cuda:0'), 'base_model.model.encoder.block.5.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0093,  0.0031,  0.0132,  ..., -0.0114,  0.0090,  0.0082],
        [ 0.0036,  0.0175, -0.0117,  ...,  0.0118, -0.0122, -0.0122],
        [-0.0102,  0.0050,  0.0037,  ...,  0.0009, -0.0026, -0.0070],
        ...,
        [ 0.0107, -0.0064,  0.0131,  ..., -0.0112,  0.0124,  0.0116],
        [-0.0078,  0.0084,  0.0060,  ...,  0.0016,  0.0031, -0.0009],
        [ 0.0077,  0.0074, -0.0095,  ...,  0.0063, -0.0104, -0.0140]],
       device='cuda:0'), 'base_model.model.encoder.block.6.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0052, -0.0190, -0.0309,  ...,  0.0368,  0.0119,  0.0045],
        [ 0.0246,  0.0021, -0.0215,  ...,  0.0109, -0.0025, -0.0697],
        [-0.0161,  0.0301, -0.0043,  ..., -0.0135, -0.0248, -0.0280],
        ...,
        [ 0.0573, -0.0103,  0.0023,  ...,  0.0115,  0.0555, -0.0128],
        [-0.0021,  0.0424, -0.0038,  ...,  0.0515,  0.0286,  0.0078],
        [-0.0103, -0.0092,  0.0028,  ...,  0.0229, -0.0282, -0.0122]],
       device='cuda:0'), 'base_model.model.encoder.block.6.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0185,  0.0261, -0.0120,  ...,  0.0008,  0.0034, -0.0148],
        [-0.0174,  0.0061, -0.0011,  ..., -0.0079,  0.0253,  0.0086],
        [ 0.0133, -0.0025, -0.0073,  ..., -0.0178,  0.0182,  0.0027],
        ...,
        [ 0.0103, -0.0069, -0.0184,  ..., -0.0013,  0.0038, -0.0002],
        [ 0.0133,  0.0024,  0.0006,  ..., -0.0015, -0.0018,  0.0221],
        [ 0.0162, -0.0035, -0.0131,  ..., -0.0185, -0.0240,  0.0108]],
       device='cuda:0'), 'base_model.model.encoder.block.6.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0123, -0.0133, -0.0479,  ...,  0.0003,  0.0049, -0.0105],
        [-0.0214,  0.0090,  0.0156,  ..., -0.0335, -0.0089, -0.0161],
        [ 0.0283, -0.0100,  0.0151,  ..., -0.0126, -0.0030, -0.0421],
        ...,
        [ 0.0319,  0.0161, -0.0138,  ...,  0.0119,  0.0346,  0.0224],
        [ 0.0075,  0.0111,  0.0164,  ...,  0.0003,  0.0058,  0.0415],
        [ 0.0122,  0.0280, -0.0096,  ...,  0.0193,  0.0315,  0.0293]],
       device='cuda:0'), 'base_model.model.encoder.block.6.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0163,  0.0121,  0.0083,  ..., -0.0032, -0.0093,  0.0055],
        [ 0.0157, -0.0087, -0.0007,  ...,  0.0037,  0.0028,  0.0137],
        [ 0.0043, -0.0024, -0.0150,  ...,  0.0119,  0.0082,  0.0046],
        ...,
        [ 0.0077, -0.0089, -0.0054,  ...,  0.0100,  0.0072,  0.0122],
        [ 0.0042, -0.0140,  0.0039,  ...,  0.0058, -0.0015, -0.0027],
        [ 0.0118, -0.0002, -0.0154,  ..., -0.0005, -0.0107, -0.0160]],
       device='cuda:0'), 'base_model.model.encoder.block.7.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0249, -0.0245,  0.0263,  ...,  0.0231,  0.0002, -0.0161],
        [-0.0284,  0.0376, -0.0152,  ...,  0.0037, -0.0273, -0.0441],
        [ 0.0552, -0.0126,  0.0192,  ...,  0.0180,  0.0117, -0.0234],
        ...,
        [ 0.0255, -0.0163,  0.0113,  ..., -0.0212, -0.0115,  0.0077],
        [-0.0227, -0.0103, -0.0073,  ...,  0.0213, -0.0048, -0.0216],
        [ 0.0301, -0.0359, -0.0198,  ...,  0.0291,  0.0122,  0.0169]],
       device='cuda:0'), 'base_model.model.encoder.block.7.layer.0.SelfAttention.q.lora_B.weight': tensor([[-3.6794e-03, -1.1577e-02, -6.4594e-03,  ..., -5.8575e-03,
          6.4691e-03,  6.1957e-03],
        [-2.4289e-02,  4.5173e-03, -2.1506e-02,  ..., -2.8095e-02,
          4.1102e-02, -1.3792e-02],
        [ 7.2917e-03, -2.7127e-02,  6.7668e-05,  ...,  7.6069e-03,
         -3.3562e-03,  7.5219e-03],
        ...,
        [-4.4854e-03, -2.3905e-03, -6.4449e-03,  ..., -7.4162e-03,
          9.0684e-03,  2.0679e-04],
        [-4.0559e-03,  1.3454e-02, -5.7965e-03,  ...,  4.8874e-04,
          1.2333e-02, -8.3721e-03],
        [-3.8800e-03, -1.0870e-02, -1.6102e-02,  ..., -2.2499e-02,
          1.5781e-02,  1.3765e-02]], device='cuda:0'), 'base_model.model.encoder.block.7.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0115,  0.0034,  0.0256,  ..., -0.0181, -0.0146, -0.0219],
        [ 0.0442, -0.0186, -0.0151,  ..., -0.0228,  0.0146,  0.0321],
        [-0.0081, -0.0252,  0.0188,  ..., -0.0297,  0.0230,  0.0064],
        ...,
        [ 0.0098,  0.0081,  0.0102,  ..., -0.0116, -0.0435,  0.0061],
        [-0.0069,  0.0040, -0.0126,  ..., -0.0046, -0.0336,  0.0175],
        [ 0.0045, -0.0129,  0.0252,  ..., -0.0328, -0.0095, -0.0250]],
       device='cuda:0'), 'base_model.model.encoder.block.7.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0095, -0.0049,  0.0090,  ..., -0.0012, -0.0114,  0.0152],
        [-0.0096, -0.0163,  0.0093,  ..., -0.0169, -0.0110,  0.0143],
        [ 0.0172,  0.0033, -0.0209,  ...,  0.0011,  0.0101, -0.0149],
        ...,
        [-0.0223, -0.0201,  0.0178,  ...,  0.0135, -0.0135, -0.0085],
        [ 0.0029, -0.0050, -0.0139,  ..., -0.0083,  0.0040, -0.0070],
        [-0.0176, -0.0048,  0.0106,  ...,  0.0100, -0.0008,  0.0078]],
       device='cuda:0'), 'base_model.model.encoder.block.8.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0547, -0.0251, -0.0336,  ...,  0.0047, -0.0150, -0.0334],
        [ 0.0297,  0.0428, -0.0070,  ...,  0.0102,  0.0042, -0.0159],
        [-0.0193,  0.0235,  0.0417,  ..., -0.0160, -0.0147,  0.0220],
        ...,
        [ 0.0081, -0.0281,  0.0176,  ..., -0.0033,  0.0308,  0.0319],
        [ 0.0157,  0.0331, -0.0299,  ...,  0.0307,  0.0073,  0.0037],
        [-0.0328, -0.0115, -0.0006,  ..., -0.0026, -0.0101,  0.0336]],
       device='cuda:0'), 'base_model.model.encoder.block.8.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0221,  0.0148,  0.0164,  ..., -0.0353, -0.0054, -0.0331],
        [ 0.0102,  0.0060, -0.0084,  ...,  0.0259,  0.0239,  0.0233],
        [-0.0187, -0.0233, -0.0057,  ..., -0.0188,  0.0055,  0.0035],
        ...,
        [ 0.0039, -0.0079, -0.0280,  ...,  0.0096,  0.0152,  0.0069],
        [-0.0040,  0.0051,  0.0182,  ..., -0.0046, -0.0015,  0.0005],
        [-0.0031,  0.0172, -0.0042,  ..., -0.0034,  0.0112,  0.0120]],
       device='cuda:0'), 'base_model.model.encoder.block.8.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0472,  0.0169, -0.0520,  ..., -0.0187,  0.0038,  0.0221],
        [-0.0078, -0.0035,  0.0112,  ..., -0.0010,  0.0342,  0.0090],
        [-0.0125,  0.0102,  0.0478,  ..., -0.0135, -0.0045,  0.0193],
        ...,
        [-0.0448, -0.0266,  0.0023,  ...,  0.0034,  0.0374,  0.0349],
        [ 0.0378,  0.0289, -0.0119,  ..., -0.0113,  0.0009, -0.0055],
        [ 0.0265, -0.0061, -0.0120,  ...,  0.0267, -0.0213, -0.0053]],
       device='cuda:0'), 'base_model.model.encoder.block.8.layer.0.SelfAttention.v.lora_B.weight': tensor([[-1.1674e-02, -1.4416e-02,  1.6741e-02,  ..., -2.0069e-02,
          1.3848e-02, -9.4953e-05],
        [ 2.8010e-03, -6.6800e-03, -1.4637e-03,  ..., -1.3755e-02,
          1.9308e-02,  1.2938e-02],
        [ 1.5742e-02, -7.6401e-03, -1.9585e-02,  ..., -7.9043e-03,
          9.8119e-03, -3.4114e-03],
        ...,
        [ 8.7199e-03, -1.3911e-02,  1.9556e-02,  ..., -1.1287e-02,
          1.3224e-02, -1.2388e-02],
        [-1.4302e-02,  2.5201e-03,  1.5830e-02,  ...,  1.1704e-02,
         -9.5458e-03, -2.3148e-03],
        [-1.2765e-02, -1.4806e-02,  3.0562e-02,  ..., -1.1230e-03,
         -2.1326e-03,  1.5665e-02]], device='cuda:0'), 'base_model.model.encoder.block.9.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0424,  0.0046, -0.0188,  ..., -0.0201,  0.0062, -0.0091],
        [ 0.0254,  0.0364,  0.0047,  ..., -0.0357, -0.0316, -0.0028],
        [-0.0442,  0.0029, -0.0119,  ..., -0.0363, -0.0380, -0.0143],
        ...,
        [ 0.0478,  0.0122,  0.0339,  ...,  0.0345, -0.0044, -0.0131],
        [ 0.0319,  0.0082, -0.0122,  ...,  0.0015, -0.0245, -0.0195],
        [ 0.0207,  0.0255,  0.0040,  ..., -0.0058, -0.0114, -0.0195]],
       device='cuda:0'), 'base_model.model.encoder.block.9.layer.0.SelfAttention.q.lora_B.weight': tensor([[-2.2374e-03,  5.3732e-03,  1.9213e-02,  ...,  1.1581e-02,
         -3.2477e-03, -4.9648e-04],
        [ 7.1751e-03, -8.7029e-03,  3.1709e-02,  ..., -2.3060e-02,
         -1.1236e-02,  1.3812e-02],
        [ 4.9562e-04, -4.9533e-03, -4.9173e-03,  ...,  2.0552e-02,
         -1.9678e-03, -3.8285e-05],
        ...,
        [ 1.4587e-02, -2.4020e-02, -1.1420e-02,  ...,  1.3391e-03,
         -4.4101e-03,  3.2557e-03],
        [-2.1797e-02,  3.1295e-02,  2.9931e-02,  ...,  1.0324e-02,
          2.1327e-02, -1.1652e-02],
        [-2.7132e-02,  5.5430e-03, -4.0748e-02,  ...,  7.3987e-03,
          1.2751e-02, -1.0383e-02]], device='cuda:0'), 'base_model.model.encoder.block.9.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0212,  0.0094, -0.0345,  ..., -0.0055,  0.0172,  0.0463],
        [-0.0218,  0.0271,  0.0145,  ..., -0.0182, -0.0750,  0.0068],
        [-0.0287, -0.0087,  0.0458,  ...,  0.0255,  0.0004, -0.0237],
        ...,
        [-0.0278,  0.0398,  0.0445,  ...,  0.0031, -0.0069, -0.0153],
        [ 0.0143, -0.0057,  0.0280,  ...,  0.0183, -0.0014,  0.0398],
        [-0.0347, -0.0115,  0.0330,  ...,  0.0281, -0.0414, -0.0309]],
       device='cuda:0'), 'base_model.model.encoder.block.9.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0175, -0.0150, -0.0187,  ..., -0.0194, -0.0235, -0.0187],
        [-0.0096,  0.0072,  0.0135,  ...,  0.0082,  0.0054,  0.0180],
        [-0.0190,  0.0146,  0.0120,  ...,  0.0130,  0.0177,  0.0227],
        ...,
        [-0.0145, -0.0077,  0.0004,  ...,  0.0100, -0.0152,  0.0171],
        [ 0.0154, -0.0045, -0.0016,  ..., -0.0170,  0.0122, -0.0131],
        [-0.0150, -0.0046, -0.0022,  ...,  0.0112, -0.0164,  0.0145]],
       device='cuda:0'), 'base_model.model.encoder.block.10.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0063, -0.0355,  0.0021,  ...,  0.0098,  0.0202, -0.0121],
        [-0.0238,  0.0037,  0.0345,  ..., -0.0088, -0.0315, -0.0167],
        [ 0.0099, -0.0115, -0.0161,  ..., -0.0287,  0.0219,  0.0324],
        ...,
        [ 0.0052, -0.0403, -0.0258,  ..., -0.0323, -0.0180,  0.0049],
        [ 0.0460, -0.0125,  0.0313,  ...,  0.0127,  0.0384, -0.0161],
        [ 0.0363, -0.0080, -0.0062,  ...,  0.0132,  0.0386, -0.0175]],
       device='cuda:0'), 'base_model.model.encoder.block.10.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0137, -0.0086, -0.0250,  ..., -0.0072, -0.0013,  0.0127],
        [ 0.0238,  0.0095, -0.0219,  ...,  0.0205,  0.0050,  0.0102],
        [ 0.0054,  0.0194, -0.0351,  ..., -0.0171,  0.0097,  0.0261],
        ...,
        [-0.0058, -0.0221,  0.0088,  ..., -0.0028,  0.0017, -0.0202],
        [ 0.0044, -0.0364,  0.0107,  ..., -0.0187, -0.0172, -0.0214],
        [-0.0165, -0.0024,  0.0094,  ..., -0.0130, -0.0179, -0.0093]],
       device='cuda:0'), 'base_model.model.encoder.block.10.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0242, -0.0273,  0.0012,  ..., -0.0284, -0.0053, -0.0016],
        [ 0.0436, -0.0018, -0.0349,  ...,  0.0164,  0.0422,  0.0017],
        [ 0.0485,  0.0392,  0.0135,  ...,  0.0272, -0.0033, -0.0040],
        ...,
        [ 0.0098,  0.0046, -0.0287,  ..., -0.0284,  0.0120,  0.0047],
        [ 0.0139, -0.0017, -0.0226,  ..., -0.0269, -0.0447,  0.0190],
        [ 0.0453, -0.0130, -0.0016,  ...,  0.0023,  0.0084,  0.0260]],
       device='cuda:0'), 'base_model.model.encoder.block.10.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0052, -0.0034,  0.0169,  ..., -0.0010,  0.0101,  0.0065],
        [-0.0235,  0.0224, -0.0020,  ...,  0.0105, -0.0222, -0.0214],
        [-0.0252,  0.0259, -0.0122,  ...,  0.0191, -0.0154, -0.0236],
        ...,
        [-0.0093,  0.0063,  0.0104,  ...,  0.0207, -0.0164, -0.0046],
        [-0.0115,  0.0047, -0.0123,  ...,  0.0146, -0.0010, -0.0085],
        [-0.0081,  0.0099,  0.0202,  ...,  0.0214, -0.0252,  0.0029]],
       device='cuda:0'), 'base_model.model.encoder.block.11.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0178, -0.0056,  0.0114,  ...,  0.0069,  0.0038,  0.0309],
        [ 0.0363,  0.0298,  0.0199,  ..., -0.0196, -0.0228,  0.0145],
        [-0.0566,  0.0105, -0.0255,  ..., -0.0212,  0.0047,  0.0039],
        ...,
        [ 0.0267,  0.0304, -0.0092,  ...,  0.0117, -0.0274, -0.0169],
        [ 0.0403,  0.0225, -0.0204,  ..., -0.0277,  0.0124, -0.0060],
        [ 0.0238, -0.0275,  0.0223,  ...,  0.0041, -0.0117,  0.0226]],
       device='cuda:0'), 'base_model.model.encoder.block.11.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 1.5251e-02, -1.3405e-02, -1.0952e-02,  ..., -3.2804e-02,
          8.0364e-03,  1.1897e-02],
        [-1.2090e-02,  1.9948e-02,  2.2969e-03,  ...,  1.9390e-02,
          5.5107e-03,  8.3068e-03],
        [-9.2829e-03,  1.3713e-02,  2.2380e-02,  ..., -3.9204e-03,
         -1.3160e-02, -3.6896e-03],
        ...,
        [-8.0247e-03,  6.4117e-03,  1.9469e-02,  ...,  1.0616e-02,
          2.5570e-04,  2.6831e-03],
        [-5.6667e-03, -5.0435e-03, -1.5369e-02,  ..., -1.1559e-02,
          1.9381e-02, -7.5156e-05],
        [ 5.1334e-03,  9.1880e-03, -2.8864e-02,  ..., -2.4764e-02,
          1.5606e-02, -2.5908e-02]], device='cuda:0'), 'base_model.model.encoder.block.11.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0133, -0.0197, -0.0365,  ..., -0.0401, -0.0523,  0.0024],
        [-0.0132,  0.0168,  0.0084,  ...,  0.0424, -0.0226,  0.0277],
        [ 0.0298,  0.0316, -0.0208,  ...,  0.0038,  0.0429,  0.0068],
        ...,
        [ 0.0310,  0.0084,  0.0016,  ...,  0.0036,  0.0146, -0.0063],
        [-0.0516, -0.0046, -0.0126,  ...,  0.0213, -0.0387, -0.0145],
        [-0.0594, -0.0328, -0.0431,  ..., -0.0225, -0.0079, -0.0229]],
       device='cuda:0'), 'base_model.model.encoder.block.11.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0250, -0.0376,  0.0345,  ...,  0.0266, -0.0233, -0.0127],
        [-0.0047,  0.0099, -0.0125,  ..., -0.0094,  0.0040,  0.0016],
        [-0.0066,  0.0164, -0.0167,  ..., -0.0113,  0.0158,  0.0106],
        ...,
        [-0.0111,  0.0109, -0.0093,  ..., -0.0007,  0.0101, -0.0061],
        [-0.0109,  0.0009, -0.0044,  ..., -0.0191, -0.0032,  0.0205],
        [-0.0100,  0.0034, -0.0044,  ..., -0.0122,  0.0104,  0.0291]],
       device='cuda:0'), 'base_model.model.encoder.block.12.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0140,  0.0012, -0.0323,  ...,  0.0490, -0.0234, -0.0137],
        [-0.0366, -0.0191,  0.0091,  ...,  0.0381,  0.0512, -0.0114],
        [ 0.0090, -0.0214, -0.0278,  ...,  0.0384,  0.0307, -0.0009],
        ...,
        [-0.0556,  0.0305,  0.0202,  ...,  0.0185, -0.0310, -0.0131],
        [-0.0287,  0.0159,  0.0026,  ...,  0.0202,  0.0368,  0.0184],
        [-0.0384,  0.0130, -0.0241,  ...,  0.0191,  0.0524,  0.0031]],
       device='cuda:0'), 'base_model.model.encoder.block.12.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0059,  0.0123,  0.0067,  ...,  0.0095,  0.0090,  0.0237],
        [ 0.0024,  0.0133,  0.0313,  ...,  0.0079,  0.0124,  0.0065],
        [ 0.0086, -0.0328, -0.0109,  ..., -0.0251, -0.0013, -0.0018],
        ...,
        [-0.0003, -0.0192,  0.0292,  ..., -0.0260,  0.0006, -0.0082],
        [ 0.0058, -0.0187, -0.0136,  ..., -0.0056, -0.0078, -0.0284],
        [ 0.0213, -0.0099, -0.0119,  ...,  0.0173,  0.0065, -0.0007]],
       device='cuda:0'), 'base_model.model.encoder.block.12.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0155,  0.0088,  0.0113,  ...,  0.0075,  0.0049,  0.0154],
        [-0.0268,  0.0203,  0.0017,  ..., -0.0269,  0.0272,  0.0151],
        [-0.0063,  0.0055, -0.0013,  ..., -0.0193,  0.0099, -0.0143],
        ...,
        [ 0.0315,  0.0117, -0.0101,  ...,  0.0034,  0.0140,  0.0136],
        [ 0.0042, -0.0548, -0.0144,  ...,  0.0041,  0.0296,  0.0246],
        [ 0.0081,  0.0255,  0.0194,  ...,  0.0020,  0.0124,  0.0246]],
       device='cuda:0'), 'base_model.model.encoder.block.12.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0034,  0.0007, -0.0139,  ..., -0.0021,  0.0076,  0.0024],
        [-0.0063, -0.0191, -0.0012,  ...,  0.0197,  0.0059,  0.0199],
        [ 0.0098,  0.0127, -0.0077,  ..., -0.0152, -0.0014, -0.0179],
        ...,
        [-0.0012, -0.0055,  0.0102,  ...,  0.0071,  0.0157,  0.0007],
        [-0.0223, -0.0310, -0.0248,  ...,  0.0279,  0.0295,  0.0272],
        [-0.0144, -0.0010, -0.0043,  ...,  0.0101,  0.0027,  0.0096]],
       device='cuda:0'), 'base_model.model.encoder.block.13.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0528,  0.0265, -0.0006,  ...,  0.0108,  0.0268, -0.0358],
        [-0.0148, -0.0179,  0.0211,  ..., -0.0519, -0.0323, -0.0162],
        [-0.0082, -0.0202, -0.0363,  ..., -0.0067,  0.0422,  0.0203],
        ...,
        [ 0.0119,  0.0068,  0.0021,  ..., -0.0284, -0.0126, -0.0077],
        [-0.0252,  0.0406, -0.0096,  ...,  0.0380, -0.0298, -0.0124],
        [ 0.0174, -0.0255,  0.0100,  ..., -0.0062, -0.0038, -0.0483]],
       device='cuda:0'), 'base_model.model.encoder.block.13.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0046,  0.0007,  0.0040,  ...,  0.0127, -0.0086,  0.0045],
        [-0.0034,  0.0012, -0.0269,  ...,  0.0077,  0.0077,  0.0079],
        [-0.0255,  0.0091,  0.0121,  ...,  0.0027, -0.0190,  0.0250],
        ...,
        [-0.0013, -0.0045,  0.0094,  ..., -0.0040, -0.0030,  0.0165],
        [-0.0094, -0.0013,  0.0098,  ...,  0.0043, -0.0060,  0.0057],
        [-0.0247,  0.0240,  0.0263,  ...,  0.0149, -0.0302,  0.0101]],
       device='cuda:0'), 'base_model.model.encoder.block.13.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0235,  0.0104,  0.0366,  ...,  0.0335,  0.0118, -0.0104],
        [-0.0169,  0.0107, -0.0293,  ..., -0.0196,  0.0248,  0.0136],
        [ 0.0046,  0.0114,  0.0013,  ..., -0.0061,  0.0208,  0.0137],
        ...,
        [-0.0008,  0.0332,  0.0251,  ..., -0.0024,  0.0390, -0.0130],
        [ 0.0226, -0.0223, -0.0490,  ..., -0.0123, -0.0358,  0.0141],
        [-0.0114, -0.0098, -0.0323,  ...,  0.0350, -0.0451,  0.0060]],
       device='cuda:0'), 'base_model.model.encoder.block.13.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 1.3388e-03, -7.8071e-03, -4.4231e-03,  ...,  6.8468e-03,
         -8.6865e-03, -6.6189e-03],
        [-8.4138e-03,  3.9207e-03, -1.7379e-02,  ..., -1.4644e-02,
          2.0210e-02,  1.8017e-02],
        [-8.6932e-03,  3.4929e-03, -8.5747e-03,  ..., -7.9256e-05,
         -7.3968e-03, -1.2425e-02],
        ...,
        [ 2.5251e-02, -1.5215e-02,  1.2540e-02,  ...,  5.3171e-04,
          5.1631e-03, -6.5597e-04],
        [-4.4899e-03, -1.2352e-02, -3.8436e-03,  ...,  4.8080e-03,
         -9.7884e-04, -6.4385e-03],
        [ 1.2952e-02, -6.3606e-03,  6.3772e-03,  ..., -9.4017e-03,
          5.4179e-03,  6.1516e-03]], device='cuda:0'), 'base_model.model.encoder.block.14.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0031, -0.0145, -0.0498,  ...,  0.0126, -0.0046,  0.0666],
        [-0.0352,  0.0089,  0.0178,  ..., -0.0369, -0.0284, -0.0033],
        [-0.0229,  0.0080, -0.0020,  ...,  0.0681,  0.0048,  0.0553],
        ...,
        [ 0.0118,  0.0116, -0.0289,  ...,  0.0028, -0.0169,  0.0415],
        [ 0.0137, -0.0024,  0.0268,  ...,  0.0435,  0.0123, -0.0126],
        [ 0.0422,  0.0337,  0.0022,  ...,  0.0373,  0.0318,  0.0344]],
       device='cuda:0'), 'base_model.model.encoder.block.14.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0116, -0.0045,  0.0094,  ...,  0.0165,  0.0021,  0.0011],
        [-0.0034,  0.0033, -0.0199,  ...,  0.0003,  0.0146,  0.0141],
        [-0.0055,  0.0080, -0.0076,  ..., -0.0197, -0.0134, -0.0145],
        ...,
        [ 0.0130, -0.0213,  0.0119,  ...,  0.0132,  0.0252,  0.0126],
        [ 0.0011, -0.0145,  0.0209,  ..., -0.0021,  0.0018, -0.0081],
        [-0.0271, -0.0154,  0.0062,  ...,  0.0095,  0.0025, -0.0112]],
       device='cuda:0'), 'base_model.model.encoder.block.14.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0097, -0.0242,  0.0057,  ...,  0.0214, -0.0358, -0.0250],
        [-0.0355, -0.0014, -0.0570,  ...,  0.0239,  0.0224, -0.0252],
        [-0.0124,  0.0291,  0.0428,  ..., -0.0319,  0.0109,  0.0128],
        ...,
        [ 0.0115, -0.0210, -0.0035,  ...,  0.0227, -0.0120,  0.0143],
        [-0.0309,  0.0038,  0.0248,  ...,  0.0052, -0.0469,  0.0364],
        [-0.0103,  0.0015, -0.0015,  ..., -0.0410,  0.0415, -0.0185]],
       device='cuda:0'), 'base_model.model.encoder.block.14.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0389, -0.0082,  0.0113,  ..., -0.0270,  0.0057,  0.0258],
        [ 0.0010,  0.0043, -0.0075,  ..., -0.0001, -0.0128, -0.0028],
        [ 0.0128,  0.0135, -0.0123,  ...,  0.0104, -0.0072, -0.0109],
        ...,
        [-0.0414, -0.0318,  0.0169,  ..., -0.0431,  0.0356,  0.0451],
        [ 0.0091, -0.0132,  0.0102,  ..., -0.0135,  0.0111, -0.0035],
        [ 0.0005,  0.0068, -0.0088,  ..., -0.0002,  0.0083, -0.0006]],
       device='cuda:0'), 'base_model.model.encoder.block.15.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0069,  0.0003, -0.0256,  ...,  0.0258, -0.0335,  0.0484],
        [-0.0216, -0.0322,  0.0204,  ...,  0.0170, -0.0222, -0.0108],
        [ 0.0008,  0.0105, -0.0380,  ..., -0.0099, -0.0394,  0.0262],
        ...,
        [-0.0015, -0.0016, -0.0252,  ..., -0.0252, -0.0368, -0.0343],
        [-0.0253,  0.0274,  0.0133,  ...,  0.0494, -0.0482,  0.0473],
        [-0.0317,  0.0028,  0.0004,  ...,  0.0471,  0.0196,  0.0111]],
       device='cuda:0'), 'base_model.model.encoder.block.15.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0266, -0.0135,  0.0290,  ..., -0.0318, -0.0019,  0.0112],
        [-0.0016,  0.0150, -0.0023,  ...,  0.0111, -0.0071, -0.0280],
        [-0.0286,  0.0020, -0.0203,  ...,  0.0246, -0.0086, -0.0261],
        ...,
        [-0.0133, -0.0072, -0.0033,  ...,  0.0133, -0.0128, -0.0120],
        [-0.0055, -0.0159, -0.0040,  ...,  0.0039, -0.0175, -0.0134],
        [-0.0024,  0.0035,  0.0067,  ...,  0.0042, -0.0094, -0.0038]],
       device='cuda:0'), 'base_model.model.encoder.block.15.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 1.7631e-03,  3.6335e-02, -2.2382e-02,  ..., -1.4592e-02,
          5.8360e-02, -1.8392e-02],
        [ 1.4876e-02,  2.0355e-02, -2.3993e-02,  ...,  1.8249e-02,
         -2.0012e-03, -2.4304e-02],
        [-1.7222e-02,  8.3348e-04, -2.3049e-02,  ..., -2.6813e-02,
          3.8505e-02, -1.4321e-02],
        ...,
        [-1.3301e-02, -1.7976e-05, -1.0527e-02,  ...,  4.7702e-02,
          2.2238e-03, -2.4194e-02],
        [-2.4366e-02, -3.5398e-02,  2.0618e-02,  ...,  3.4942e-03,
         -4.9058e-02,  8.0171e-03],
        [-3.0048e-03,  3.7638e-02, -3.8622e-03,  ..., -5.0610e-03,
          5.2482e-02, -1.0379e-02]], device='cuda:0'), 'base_model.model.encoder.block.15.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 2.5725e-03,  3.5926e-03,  1.9242e-03,  ..., -1.1365e-02,
         -2.6424e-04, -4.3521e-03],
        [-2.1243e-03, -1.0515e-02,  4.6888e-03,  ..., -3.1430e-03,
         -6.3100e-03, -6.4766e-03],
        [-1.0521e-02, -1.0788e-02, -6.9606e-03,  ...,  1.1522e-02,
          3.2694e-03, -1.3453e-02],
        ...,
        [-3.5505e-05, -2.4789e-02,  5.3724e-03,  ...,  1.2047e-02,
         -5.5704e-03, -1.3944e-02],
        [-1.4204e-02, -2.3716e-02, -2.5142e-03,  ...,  1.1288e-02,
          1.0336e-02, -1.5918e-02],
        [-9.6600e-03, -4.7291e-04, -1.8597e-02,  ...,  1.7493e-02,
          1.8787e-02, -1.6483e-02]], device='cuda:0'), 'base_model.model.encoder.block.16.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0099, -0.0129,  0.0517,  ..., -0.0158,  0.0089,  0.0313],
        [ 0.0373, -0.0277,  0.0096,  ..., -0.0250, -0.0250,  0.0282],
        [-0.0122, -0.0129, -0.0159,  ..., -0.0003,  0.0164,  0.0209],
        ...,
        [ 0.0360, -0.0043,  0.0127,  ...,  0.0088, -0.0333, -0.0175],
        [ 0.0071, -0.0533, -0.0338,  ...,  0.0417, -0.0052,  0.0208],
        [ 0.0142, -0.0381,  0.0059,  ..., -0.0328,  0.0163,  0.0182]],
       device='cuda:0'), 'base_model.model.encoder.block.16.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0117, -0.0307, -0.0021,  ...,  0.0123, -0.0253, -0.0016],
        [-0.0085,  0.0046,  0.0009,  ..., -0.0039,  0.0105, -0.0178],
        [ 0.0037,  0.0178,  0.0198,  ..., -0.0141,  0.0087,  0.0110],
        ...,
        [-0.0101,  0.0057,  0.0065,  ...,  0.0019,  0.0080, -0.0162],
        [ 0.0104,  0.0138,  0.0215,  ..., -0.0142,  0.0151,  0.0108],
        [ 0.0186,  0.0051,  0.0312,  ..., -0.0250,  0.0252,  0.0021]],
       device='cuda:0'), 'base_model.model.encoder.block.16.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0258, -0.0153, -0.0037,  ..., -0.0136, -0.0433,  0.0465],
        [-0.0139, -0.0489, -0.0567,  ...,  0.0056,  0.0010,  0.0062],
        [ 0.0086,  0.0541,  0.0437,  ..., -0.0202,  0.0503, -0.0420],
        ...,
        [ 0.0304,  0.0422, -0.0691,  ..., -0.0162,  0.0146,  0.0323],
        [-0.0310,  0.0305,  0.0252,  ...,  0.0340, -0.0040, -0.0321],
        [ 0.0063,  0.0173, -0.0269,  ...,  0.0041, -0.0490,  0.0162]],
       device='cuda:0'), 'base_model.model.encoder.block.16.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0372, -0.0004, -0.0395,  ..., -0.0252, -0.0212,  0.0400],
        [ 0.0557,  0.0030, -0.0577,  ..., -0.0191, -0.0292,  0.0516],
        [-0.0057,  0.0290,  0.0045,  ...,  0.0348, -0.0138, -0.0093],
        ...,
        [-0.0273, -0.0235,  0.0251,  ..., -0.0232,  0.0207, -0.0205],
        [ 0.0374,  0.0416, -0.0401,  ...,  0.0239, -0.0532,  0.0429],
        [-0.0201, -0.0159,  0.0268,  ..., -0.0145,  0.0342, -0.0210]],
       device='cuda:0'), 'base_model.model.encoder.block.17.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0124,  0.0274,  0.0223,  ..., -0.0135,  0.0262,  0.0426],
        [-0.0069,  0.0114,  0.0237,  ...,  0.0522,  0.0042,  0.0107],
        [ 0.0016, -0.0762, -0.0267,  ...,  0.0016,  0.0416, -0.0379],
        ...,
        [-0.0107, -0.0051,  0.0061,  ...,  0.0077, -0.0267, -0.0538],
        [-0.0166, -0.0188, -0.0361,  ..., -0.0321,  0.0282, -0.0272],
        [ 0.0195, -0.0308,  0.0041,  ..., -0.0343, -0.0432, -0.0337]],
       device='cuda:0'), 'base_model.model.encoder.block.17.layer.0.SelfAttention.q.lora_B.weight': tensor([[-1.4683e-02, -9.0315e-03,  1.3334e-02,  ...,  6.3660e-03,
          3.7701e-03,  1.8145e-02],
        [ 2.2848e-03, -4.4163e-03,  1.6257e-02,  ...,  8.8848e-03,
          1.0144e-02,  6.2522e-03],
        [ 1.5033e-02,  1.0294e-02, -1.7981e-02,  ..., -8.9641e-03,
         -2.1745e-02, -2.4771e-02],
        ...,
        [ 2.1272e-04, -7.2553e-03, -2.3124e-06,  ...,  3.4841e-03,
         -1.0348e-02, -2.2534e-02],
        [-2.2342e-03,  1.8137e-02, -6.5606e-03,  ...,  2.7981e-03,
         -8.4023e-03,  1.4239e-02],
        [-3.0212e-03,  3.7128e-03, -1.7892e-02,  ...,  1.1220e-02,
         -2.2024e-02, -8.1391e-03]], device='cuda:0'), 'base_model.model.encoder.block.17.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0019, -0.0137, -0.0012,  ..., -0.0406, -0.0212,  0.0466],
        [ 0.0227, -0.0154,  0.0074,  ...,  0.0155, -0.0104,  0.0346],
        [-0.0455,  0.0035,  0.0101,  ..., -0.0231, -0.0183, -0.0431],
        ...,
        [-0.0447,  0.0181,  0.0305,  ...,  0.0417, -0.0317, -0.0054],
        [ 0.0427, -0.0357, -0.0128,  ..., -0.0214,  0.0126,  0.0405],
        [-0.0563,  0.0184, -0.0125,  ...,  0.0406, -0.0077, -0.0211]],
       device='cuda:0'), 'base_model.model.encoder.block.17.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0090, -0.0032, -0.0055,  ..., -0.0054, -0.0003,  0.0061],
        [ 0.0058,  0.0085, -0.0087,  ..., -0.0043,  0.0099,  0.0082],
        [ 0.0087,  0.0208, -0.0196,  ..., -0.0181,  0.0206,  0.0036],
        ...,
        [-0.0395, -0.0220,  0.0149,  ...,  0.0223, -0.0172,  0.0448],
        [-0.0227, -0.0071,  0.0050,  ...,  0.0142, -0.0116,  0.0192],
        [-0.0102,  0.0058, -0.0205,  ..., -0.0027,  0.0044,  0.0175]],
       device='cuda:0'), 'base_model.model.encoder.block.18.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0241,  0.0187,  0.0110,  ...,  0.0043, -0.0370,  0.0384],
        [ 0.0038,  0.0020, -0.0249,  ..., -0.0397,  0.0138, -0.0315],
        [-0.0226, -0.0243, -0.0236,  ...,  0.0053, -0.0367, -0.0319],
        ...,
        [ 0.0032,  0.0287, -0.0235,  ..., -0.0466, -0.0053,  0.0106],
        [-0.0264, -0.0175, -0.0500,  ..., -0.0234,  0.0288, -0.0165],
        [-0.0156,  0.0258,  0.0038,  ..., -0.0575, -0.0257, -0.0088]],
       device='cuda:0'), 'base_model.model.encoder.block.18.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0184, -0.0091, -0.0198,  ..., -0.0144, -0.0019, -0.0017],
        [-0.0016,  0.0060, -0.0113,  ...,  0.0096, -0.0067,  0.0017],
        [-0.0023, -0.0006, -0.0505,  ..., -0.0005, -0.0123,  0.0116],
        ...,
        [-0.0247,  0.0048,  0.0250,  ...,  0.0066, -0.0144,  0.0116],
        [ 0.0060, -0.0079, -0.0068,  ..., -0.0094,  0.0171, -0.0095],
        [-0.0086, -0.0147,  0.0030,  ..., -0.0247,  0.0092, -0.0146]],
       device='cuda:0'), 'base_model.model.encoder.block.18.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0043,  0.0391, -0.0416,  ..., -0.0416,  0.0277,  0.0104],
        [-0.0030,  0.0658, -0.0103,  ..., -0.0245,  0.0268,  0.0276],
        [ 0.0047,  0.0221, -0.0067,  ...,  0.0026,  0.0197,  0.0375],
        ...,
        [-0.0245, -0.0708,  0.0604,  ...,  0.0216,  0.0002, -0.0149],
        [ 0.0146, -0.0349,  0.0189,  ..., -0.0136, -0.0235,  0.0123],
        [-0.0083, -0.0331,  0.0403,  ...,  0.0157, -0.0047, -0.0127]],
       device='cuda:0'), 'base_model.model.encoder.block.18.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0209, -0.0291, -0.0244,  ...,  0.0253,  0.0127, -0.0248],
        [ 0.0053,  0.0060,  0.0070,  ..., -0.0051, -0.0021, -0.0103],
        [ 0.0148,  0.0031,  0.0126,  ..., -0.0117, -0.0017, -0.0147],
        ...,
        [ 0.0200,  0.0158,  0.0231,  ..., -0.0149, -0.0121,  0.0300],
        [-0.0008, -0.0023, -0.0031,  ...,  0.0107,  0.0033,  0.0051],
        [ 0.0240,  0.0208,  0.0184,  ..., -0.0156, -0.0005,  0.0188]],
       device='cuda:0'), 'base_model.model.encoder.block.19.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0249, -0.0210, -0.0265,  ...,  0.0108, -0.0111, -0.0171],
        [-0.0351,  0.0088, -0.0190,  ...,  0.0171,  0.0221,  0.0042],
        [ 0.0030,  0.0014, -0.0108,  ...,  0.0105,  0.0679,  0.0146],
        ...,
        [ 0.0347, -0.0230,  0.0132,  ..., -0.0003, -0.0530,  0.0031],
        [ 0.0138,  0.0345,  0.0108,  ...,  0.0178,  0.0252,  0.0180],
        [ 0.0034,  0.0017, -0.0196,  ..., -0.0089, -0.0449, -0.0323]],
       device='cuda:0'), 'base_model.model.encoder.block.19.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0205, -0.0094, -0.0139,  ...,  0.0070, -0.0126,  0.0226],
        [-0.0010, -0.0070, -0.0134,  ...,  0.0123, -0.0060,  0.0030],
        [-0.0007, -0.0004, -0.0068,  ...,  0.0005,  0.0003,  0.0037],
        ...,
        [ 0.0378, -0.0430, -0.0296,  ...,  0.0429, -0.0451,  0.0347],
        [-0.0320,  0.0106,  0.0203,  ..., -0.0211,  0.0239, -0.0175],
        [ 0.0210, -0.0117, -0.0150,  ...,  0.0160, -0.0256,  0.0168]],
       device='cuda:0'), 'base_model.model.encoder.block.19.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 1.4258e-02, -2.0162e-02,  1.4808e-02,  ...,  6.3350e-02,
          3.1473e-02, -2.1102e-02],
        [ 3.0901e-03, -4.1341e-02,  2.0951e-02,  ...,  1.8360e-02,
         -6.7527e-03,  1.2380e-02],
        [ 2.9749e-02,  5.5730e-05, -6.0016e-03,  ...,  4.4938e-02,
          6.8638e-03, -1.5086e-02],
        ...,
        [-6.1253e-03,  1.3493e-03,  1.6801e-02,  ...,  5.0317e-03,
         -5.3136e-03,  1.7972e-02],
        [ 1.9262e-02, -1.0739e-03,  5.0379e-02,  ...,  6.5103e-02,
          2.6762e-02, -1.0854e-02],
        [ 2.7448e-02,  2.8939e-03,  4.7538e-02,  ...,  4.8242e-02,
          2.4452e-02, -2.7716e-02]], device='cuda:0'), 'base_model.model.encoder.block.19.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0130, -0.0170, -0.0146,  ..., -0.0144, -0.0169, -0.0175],
        [-0.0181, -0.0188, -0.0173,  ..., -0.0193, -0.0192, -0.0150],
        [ 0.0139,  0.0166,  0.0089,  ...,  0.0140,  0.0107,  0.0067],
        ...,
        [-0.0119, -0.0102, -0.0134,  ..., -0.0145, -0.0182, -0.0151],
        [ 0.0106,  0.0094,  0.0102,  ...,  0.0079,  0.0062,  0.0125],
        [-0.0259, -0.0304, -0.0267,  ..., -0.0262, -0.0288, -0.0286]],
       device='cuda:0'), 'base_model.model.encoder.block.20.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0136, -0.0010, -0.0112,  ...,  0.0021, -0.0050,  0.0332],
        [ 0.0083, -0.0270, -0.0023,  ...,  0.0004, -0.0469,  0.0221],
        [-0.0129, -0.0622,  0.0074,  ...,  0.0102, -0.0384,  0.0262],
        ...,
        [-0.0207, -0.0472,  0.0020,  ...,  0.0109, -0.0551,  0.0010],
        [-0.0171,  0.0238, -0.0192,  ...,  0.0388,  0.0184, -0.0068],
        [ 0.0377, -0.0111,  0.0170,  ..., -0.0148, -0.0103,  0.0415]],
       device='cuda:0'), 'base_model.model.encoder.block.20.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0201, -0.0002, -0.0038,  ..., -0.0076,  0.0105, -0.0084],
        [-0.0349,  0.0177, -0.0228,  ...,  0.0044,  0.0148, -0.0253],
        [-0.0230,  0.0061,  0.0187,  ...,  0.0196, -0.0039,  0.0043],
        ...,
        [-0.0062,  0.0211, -0.0158,  ..., -0.0114,  0.0234, -0.0168],
        [-0.0158, -0.0012,  0.0284,  ...,  0.0266, -0.0246,  0.0207],
        [-0.0249,  0.0377, -0.0102,  ...,  0.0059,  0.0032, -0.0116]],
       device='cuda:0'), 'base_model.model.encoder.block.20.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0300,  0.0260,  0.0026,  ...,  0.0304,  0.0345,  0.0474],
        [-0.0201, -0.0045,  0.0204,  ..., -0.0125, -0.0221, -0.0285],
        [-0.0180, -0.0343,  0.0173,  ...,  0.0428,  0.0268,  0.0424],
        ...,
        [ 0.0010, -0.0441, -0.0271,  ...,  0.0089,  0.0099, -0.0180],
        [-0.0098, -0.0109,  0.0484,  ...,  0.0118,  0.0068, -0.0187],
        [-0.0206, -0.0263, -0.0238,  ...,  0.0390,  0.0060,  0.0163]],
       device='cuda:0'), 'base_model.model.encoder.block.20.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0201, -0.0216,  0.0119,  ...,  0.0183, -0.0170,  0.0124],
        [ 0.0130, -0.0100,  0.0180,  ...,  0.0223,  0.0051,  0.0060],
        [-0.0278,  0.0233, -0.0264,  ..., -0.0253,  0.0271, -0.0330],
        ...,
        [-0.0052, -0.0034,  0.0056,  ...,  0.0010,  0.0099,  0.0036],
        [-0.0003, -0.0029, -0.0112,  ..., -0.0089, -0.0206,  0.0131],
        [ 0.0036, -0.0245,  0.0003,  ..., -0.0018, -0.0305,  0.0315]],
       device='cuda:0'), 'base_model.model.encoder.block.21.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0078, -0.0170,  0.0393,  ..., -0.0284,  0.0189,  0.0169],
        [-0.0003,  0.0058, -0.0220,  ...,  0.0204, -0.0052, -0.0126],
        [ 0.0164,  0.0175, -0.0063,  ...,  0.0019,  0.0210, -0.0453],
        ...,
        [ 0.0046,  0.0172,  0.0201,  ..., -0.0191,  0.0161,  0.0512],
        [ 0.0214, -0.0041,  0.0220,  ..., -0.0172,  0.0319,  0.0360],
        [-0.0295,  0.0370, -0.0062,  ...,  0.0288,  0.0298,  0.0034]],
       device='cuda:0'), 'base_model.model.encoder.block.21.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0021,  0.0006,  0.0097,  ..., -0.0011,  0.0026, -0.0374],
        [ 0.0022, -0.0127, -0.0116,  ...,  0.0035,  0.0007,  0.0061],
        [-0.0159,  0.0103,  0.0079,  ..., -0.0159, -0.0147, -0.0258],
        ...,
        [-0.0059,  0.0005, -0.0015,  ..., -0.0041, -0.0017, -0.0200],
        [-0.0226,  0.0153,  0.0165,  ..., -0.0202, -0.0170,  0.0057],
        [-0.0090,  0.0059,  0.0104,  ..., -0.0058, -0.0050, -0.0039]],
       device='cuda:0'), 'base_model.model.encoder.block.21.layer.0.SelfAttention.v.lora_A.weight': tensor([[-2.9747e-03,  1.7645e-05, -1.3959e-02,  ...,  1.6702e-02,
          2.5911e-02, -1.3416e-02],
        [ 1.5099e-02,  3.0057e-02, -2.4511e-02,  ..., -9.5109e-03,
         -1.4320e-02, -3.5647e-02],
        [ 8.3559e-03, -2.1729e-02,  6.0549e-03,  ...,  9.1296e-04,
          7.1130e-03,  6.5738e-03],
        ...,
        [ 2.6930e-02, -1.5346e-02,  8.9250e-03,  ...,  2.1033e-02,
         -1.1156e-02,  1.5283e-02],
        [ 3.1569e-02, -1.3637e-02, -4.1950e-02,  ..., -1.9484e-02,
          4.6394e-02, -1.6220e-02],
        [ 3.5849e-03,  3.0520e-02, -2.6064e-02,  ..., -2.2740e-02,
          4.4686e-02,  4.4240e-02]], device='cuda:0'), 'base_model.model.encoder.block.21.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0174, -0.0201, -0.0169,  ...,  0.0173,  0.0179,  0.0225],
        [ 0.0099, -0.0035, -0.0075,  ..., -0.0073, -0.0029, -0.0030],
        [ 0.0022, -0.0011, -0.0103,  ...,  0.0097,  0.0104,  0.0053],
        ...,
        [ 0.0152, -0.0175,  0.0037,  ...,  0.0109, -0.0065,  0.0123],
        [-0.0031, -0.0083,  0.0029,  ...,  0.0136, -0.0016, -0.0011],
        [-0.0010,  0.0170, -0.0216,  ..., -0.0187,  0.0204,  0.0015]],
       device='cuda:0'), 'base_model.model.encoder.block.22.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0314, -0.0045, -0.0207,  ..., -0.0543, -0.0126,  0.0270],
        [ 0.0462,  0.0351, -0.0017,  ...,  0.0194,  0.0221,  0.0235],
        [ 0.0180,  0.0013,  0.0002,  ..., -0.0235, -0.0701, -0.0042],
        ...,
        [-0.0203, -0.0024,  0.0114,  ..., -0.0052, -0.0093, -0.0337],
        [-0.0250,  0.0583,  0.0322,  ...,  0.0131,  0.0062, -0.0221],
        [-0.0365, -0.0355,  0.0022,  ..., -0.0091, -0.0091, -0.0029]],
       device='cuda:0'), 'base_model.model.encoder.block.22.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0084,  0.0145, -0.0058,  ...,  0.0047,  0.0110, -0.0105],
        [ 0.0115,  0.0224,  0.0182,  ..., -0.0159, -0.0033, -0.0011],
        [-0.0078,  0.0098, -0.0079,  ...,  0.0062,  0.0060,  0.0024],
        ...,
        [ 0.0142, -0.0038,  0.0094,  ..., -0.0021, -0.0138,  0.0140],
        [ 0.0071, -0.0059, -0.0077,  ...,  0.0050, -0.0060,  0.0048],
        [ 0.0117, -0.0122, -0.0040,  ...,  0.0117, -0.0074,  0.0111]],
       device='cuda:0'), 'base_model.model.encoder.block.22.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0403,  0.0288,  0.0345,  ..., -0.0313,  0.0456,  0.0290],
        [ 0.0169, -0.0029, -0.0118,  ..., -0.0086,  0.0247, -0.0073],
        [-0.0182, -0.0154, -0.0023,  ...,  0.0213,  0.0123, -0.0090],
        ...,
        [ 0.0398, -0.0134, -0.0142,  ..., -0.0308,  0.0373,  0.0003],
        [-0.0289,  0.0193, -0.0009,  ...,  0.0243, -0.0158, -0.0300],
        [-0.0136, -0.0490, -0.0062,  ...,  0.0037, -0.0265, -0.0094]],
       device='cuda:0'), 'base_model.model.encoder.block.22.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 1.1087e-05, -2.0695e-02, -1.8504e-02,  ..., -2.3577e-02,
         -1.6880e-02,  2.1301e-02],
        [-2.5760e-02, -1.2344e-02, -8.9854e-03,  ...,  2.8194e-03,
         -1.9121e-02,  9.2568e-03],
        [-2.2456e-03, -1.0488e-02, -7.2482e-03,  ..., -1.6379e-02,
         -7.3226e-03,  1.2839e-02],
        ...,
        [-8.2132e-03, -1.3901e-02, -1.9611e-02,  ..., -1.4056e-02,
         -1.2747e-02,  1.2911e-02],
        [ 6.6918e-04, -9.2962e-03, -1.1142e-02,  ..., -8.8167e-03,
         -1.0819e-02,  1.0372e-02],
        [-6.8010e-03,  1.8507e-02,  2.1169e-02,  ...,  1.6902e-02,
          2.3338e-02, -1.5173e-02]], device='cuda:0'), 'base_model.model.encoder.block.23.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0287,  0.0413, -0.0387,  ...,  0.0297, -0.0248,  0.0058],
        [ 0.0030,  0.0372,  0.0231,  ..., -0.0069, -0.0422,  0.0422],
        [ 0.0030,  0.0138, -0.0341,  ...,  0.0274, -0.0051, -0.0047],
        ...,
        [-0.0029, -0.0164, -0.0410,  ...,  0.0089, -0.0333, -0.0027],
        [-0.0257, -0.0437,  0.0370,  ..., -0.0213,  0.0096, -0.0510],
        [-0.0032, -0.0417, -0.0039,  ..., -0.0076,  0.0019, -0.0326]],
       device='cuda:0'), 'base_model.model.encoder.block.23.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0257, -0.0177, -0.0280,  ..., -0.0275,  0.0045,  0.0162],
        [ 0.0056,  0.0036,  0.0022,  ...,  0.0057, -0.0219, -0.0013],
        [ 0.0139,  0.0069,  0.0139,  ...,  0.0051,  0.0040, -0.0113],
        ...,
        [ 0.0008, -0.0016, -0.0095,  ...,  0.0090,  0.0071,  0.0025],
        [-0.0113, -0.0092, -0.0026,  ..., -0.0123, -0.0069,  0.0113],
        [ 0.0099,  0.0207,  0.0191,  ...,  0.0130, -0.0135, -0.0131]],
       device='cuda:0'), 'base_model.model.encoder.block.23.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0257, -0.0324,  0.0004,  ..., -0.0116,  0.0031,  0.0029],
        [-0.0007, -0.0106, -0.0271,  ..., -0.0073,  0.0254,  0.0318],
        [ 0.0251, -0.0242,  0.0158,  ...,  0.0124, -0.0365,  0.0444],
        ...,
        [ 0.0357,  0.0193,  0.0192,  ...,  0.0061, -0.0062, -0.0068],
        [-0.0283, -0.0168,  0.0212,  ...,  0.0222,  0.0310, -0.0504],
        [ 0.0458,  0.0061,  0.0074,  ..., -0.0495, -0.0247,  0.0176]],
       device='cuda:0'), 'base_model.model.encoder.block.23.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0123, -0.0024, -0.0170,  ..., -0.0145,  0.0109,  0.0031],
        [ 0.0129, -0.0127, -0.0100,  ..., -0.0079,  0.0134, -0.0099],
        [ 0.0170, -0.0094, -0.0102,  ..., -0.0114,  0.0088, -0.0183],
        ...,
        [ 0.0126, -0.0147, -0.0172,  ..., -0.0108,  0.0128, -0.0092],
        [ 0.0133, -0.0093, -0.0106,  ..., -0.0100,  0.0082, -0.0139],
        [ 0.0047, -0.0057, -0.0064,  ..., -0.0078,  0.0074,  0.0181]],
       device='cuda:0'), 'base_model.model.decoder.block.0.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0189, -0.0005,  0.0231,  ..., -0.0206, -0.0043, -0.0048],
        [-0.0072,  0.0131,  0.0362,  ...,  0.0150, -0.0252,  0.0271],
        [ 0.0177,  0.0022, -0.0043,  ...,  0.0060,  0.0091, -0.0020],
        ...,
        [-0.0144,  0.0091,  0.0296,  ..., -0.0143,  0.0180, -0.0202],
        [-0.0311,  0.0043,  0.0205,  ..., -0.0082, -0.0271,  0.0369],
        [ 0.0101,  0.0102, -0.0180,  ..., -0.0206,  0.0186,  0.0433]],
       device='cuda:0'), 'base_model.model.decoder.block.0.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0065, -0.0055,  0.0070,  ...,  0.0076, -0.0065, -0.0067],
        [-0.0073,  0.0057, -0.0076,  ..., -0.0072,  0.0075,  0.0079],
        [-0.0062,  0.0056, -0.0066,  ..., -0.0065,  0.0065,  0.0077],
        ...,
        [ 0.0048,  0.0183,  0.0174,  ..., -0.0067,  0.0044,  0.0170],
        [-0.0057,  0.0079,  0.0116,  ..., -0.0142,  0.0133,  0.0104],
        [-0.0040, -0.0191, -0.0187,  ...,  0.0082, -0.0056, -0.0184]],
       device='cuda:0'), 'base_model.model.decoder.block.0.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0238,  0.0030,  0.0195,  ..., -0.0004, -0.0226, -0.0122],
        [-0.0099, -0.0346, -0.0219,  ...,  0.0218,  0.0104, -0.0102],
        [-0.0086, -0.0257,  0.0187,  ...,  0.0245,  0.0112, -0.0370],
        ...,
        [ 0.0131, -0.0005, -0.0091,  ..., -0.0176, -0.0367, -0.0028],
        [ 0.0249,  0.0116,  0.0185,  ..., -0.0210,  0.0073,  0.0264],
        [-0.0072, -0.0039, -0.0250,  ...,  0.0153,  0.0267, -0.0015]],
       device='cuda:0'), 'base_model.model.decoder.block.0.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0131,  0.0152,  0.0148,  ...,  0.0143, -0.0135, -0.0138],
        [-0.0036,  0.0048,  0.0048,  ...,  0.0043, -0.0045, -0.0045],
        [-0.0127,  0.0133,  0.0084,  ...,  0.0093, -0.0077, -0.0085],
        ...,
        [-0.0257,  0.0265,  0.0275,  ...,  0.0272, -0.0283, -0.0266],
        [ 0.0154, -0.0107, -0.0156,  ..., -0.0120,  0.0129,  0.0101],
        [-0.0262,  0.0268,  0.0253,  ...,  0.0273, -0.0244, -0.0263]],
       device='cuda:0'), 'base_model.model.decoder.block.0.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0221, -0.0252,  0.0064,  ..., -0.0018, -0.0022, -0.0222],
        [-0.0064,  0.0094, -0.0091,  ...,  0.0109, -0.0069, -0.0229],
        [-0.0291,  0.0128,  0.0075,  ..., -0.0042, -0.0206,  0.0037],
        ...,
        [-0.0383, -0.0205,  0.0227,  ..., -0.0033, -0.0002,  0.0206],
        [ 0.0220,  0.0185, -0.0156,  ...,  0.0298,  0.0298, -0.0049],
        [ 0.0445, -0.0044, -0.0248,  ...,  0.0415, -0.0142,  0.0047]],
       device='cuda:0'), 'base_model.model.decoder.block.0.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0167,  0.0058,  0.0135,  ...,  0.0126, -0.0094, -0.0168],
        [ 0.0015,  0.0061,  0.0088,  ...,  0.0045, -0.0071, -0.0033],
        [-0.0220,  0.0031, -0.0131,  ..., -0.0210,  0.0146,  0.0125],
        ...,
        [ 0.0155, -0.0182,  0.0175,  ...,  0.0183, -0.0085, -0.0135],
        [ 0.0049,  0.0088,  0.0103,  ...,  0.0133, -0.0007, -0.0076],
        [-0.0018, -0.0202, -0.0031,  ...,  0.0013,  0.0120,  0.0030]],
       device='cuda:0'), 'base_model.model.decoder.block.0.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0206, -0.0007,  0.0363,  ..., -0.0155,  0.0125,  0.0302],
        [ 0.0071, -0.0351, -0.0071,  ..., -0.0167,  0.0122, -0.0117],
        [ 0.0421, -0.0014, -0.0129,  ..., -0.0072, -0.0114, -0.0469],
        ...,
        [-0.0363, -0.0197,  0.0016,  ..., -0.0347, -0.0018,  0.0446],
        [ 0.0193,  0.0251, -0.0119,  ...,  0.0415,  0.0172, -0.0267],
        [ 0.0162,  0.0387, -0.0214,  ..., -0.0331,  0.0331, -0.0025]],
       device='cuda:0'), 'base_model.model.decoder.block.0.layer.1.EncDecAttention.v.lora_B.weight': tensor([[-0.0207,  0.0257,  0.0249,  ..., -0.0258,  0.0105, -0.0223],
        [-0.0098,  0.0059,  0.0131,  ..., -0.0124, -0.0172, -0.0154],
        [-0.0001, -0.0220,  0.0026,  ...,  0.0013, -0.0207,  0.0071],
        ...,
        [-0.0171,  0.0202,  0.0187,  ..., -0.0191, -0.0165, -0.0220],
        [ 0.0091, -0.0049, -0.0022,  ...,  0.0102,  0.0082,  0.0143],
        [ 0.0122, -0.0073, -0.0114,  ...,  0.0135, -0.0045,  0.0117]],
       device='cuda:0'), 'base_model.model.decoder.block.1.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0062,  0.0340, -0.0126,  ..., -0.0317, -0.0148,  0.0114],
        [ 0.0016,  0.0201, -0.0259,  ..., -0.0081, -0.0180,  0.0036],
        [-0.0196, -0.0027,  0.0157,  ..., -0.0040, -0.0164, -0.0303],
        ...,
        [-0.0307, -0.0401,  0.0096,  ..., -0.0108,  0.0102, -0.0025],
        [ 0.0146,  0.0296,  0.0074,  ...,  0.0368,  0.0319,  0.0163],
        [ 0.0212,  0.0060, -0.0247,  ...,  0.0213,  0.0096,  0.0179]],
       device='cuda:0'), 'base_model.model.decoder.block.1.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0083,  0.0084, -0.0099,  ...,  0.0071, -0.0083, -0.0079],
        [-0.0076, -0.0083,  0.0089,  ..., -0.0073,  0.0083,  0.0083],
        [-0.0078, -0.0082,  0.0091,  ..., -0.0071,  0.0082,  0.0078],
        ...,
        [ 0.0186, -0.0079, -0.0116,  ..., -0.0011,  0.0153,  0.0145],
        [ 0.0055, -0.0129,  0.0063,  ..., -0.0055, -0.0006,  0.0057],
        [-0.0068,  0.0094,  0.0049,  ..., -0.0214, -0.0105, -0.0143]],
       device='cuda:0'), 'base_model.model.decoder.block.1.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0069,  0.0235, -0.0279,  ...,  0.0227, -0.0086,  0.0170],
        [-0.0380, -0.0010,  0.0127,  ..., -0.0040, -0.0202, -0.0155],
        [ 0.0048, -0.0332,  0.0042,  ...,  0.0085, -0.0161, -0.0132],
        ...,
        [ 0.0162, -0.0120, -0.0343,  ...,  0.0348, -0.0002,  0.0094],
        [ 0.0057, -0.0085, -0.0151,  ..., -0.0225, -0.0477, -0.0247],
        [ 0.0267, -0.0135, -0.0142,  ...,  0.0340, -0.0055, -0.0230]],
       device='cuda:0'), 'base_model.model.decoder.block.1.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0185, -0.0192,  0.0192,  ...,  0.0191, -0.0146,  0.0189],
        [-0.0069, -0.0093,  0.0093,  ...,  0.0110, -0.0066,  0.0097],
        [-0.0158, -0.0179,  0.0180,  ...,  0.0171, -0.0151,  0.0170],
        ...,
        [ 0.0105,  0.0053, -0.0043,  ..., -0.0051,  0.0076, -0.0062],
        [-0.0061, -0.0023,  0.0024,  ...,  0.0026, -0.0050,  0.0019],
        [-0.0016, -0.0042,  0.0057,  ...,  0.0055,  0.0083,  0.0062]],
       device='cuda:0'), 'base_model.model.decoder.block.1.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0090,  0.0097,  0.0331,  ...,  0.0069, -0.0390, -0.0076],
        [-0.0125,  0.0190, -0.0041,  ...,  0.0240, -0.0379,  0.0208],
        [ 0.0228, -0.0280,  0.0044,  ...,  0.0122,  0.0094,  0.0115],
        ...,
        [-0.0252,  0.0296,  0.0183,  ..., -0.0186,  0.0372,  0.0183],
        [-0.0111, -0.0163,  0.0142,  ...,  0.0146, -0.0015, -0.0034],
        [ 0.0198,  0.0215, -0.0137,  ..., -0.0140,  0.0162,  0.0132]],
       device='cuda:0'), 'base_model.model.decoder.block.1.layer.1.EncDecAttention.q.lora_B.weight': tensor([[-0.0008,  0.0053,  0.0051,  ...,  0.0054, -0.0029,  0.0046],
        [ 0.0004,  0.0072,  0.0072,  ...,  0.0089,  0.0025,  0.0062],
        [ 0.0051, -0.0091, -0.0187,  ..., -0.0199,  0.0004, -0.0147],
        ...,
        [ 0.0021,  0.0219,  0.0079,  ...,  0.0080,  0.0092,  0.0068],
        [-0.0025, -0.0209, -0.0022,  ..., -0.0014, -0.0061, -0.0011],
        [ 0.0134,  0.0163, -0.0054,  ..., -0.0052,  0.0140, -0.0088]],
       device='cuda:0'), 'base_model.model.decoder.block.1.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0112,  0.0366, -0.0094,  ..., -0.0264,  0.0193,  0.0159],
        [-0.0172, -0.0171, -0.0273,  ...,  0.0062, -0.0069, -0.0036],
        [ 0.0194,  0.0024, -0.0140,  ..., -0.0079,  0.0263, -0.0028],
        ...,
        [ 0.0128, -0.0109, -0.0209,  ..., -0.0019, -0.0316, -0.0220],
        [ 0.0199,  0.0126,  0.0240,  ..., -0.0239,  0.0473,  0.0146],
        [ 0.0186,  0.0341, -0.0052,  ..., -0.0248,  0.0462,  0.0117]],
       device='cuda:0'), 'base_model.model.decoder.block.1.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 2.3198e-02, -2.1747e-02,  2.1701e-02,  ..., -2.1566e-02,
          2.4150e-02,  2.1774e-02],
        [-1.6284e-03,  7.0164e-04, -1.1504e-03,  ...,  4.7365e-03,
         -9.8117e-05, -6.4463e-03],
        [ 1.5113e-02, -1.2611e-02,  1.4666e-02,  ..., -9.9986e-03,
          1.1651e-02,  2.0736e-03],
        ...,
        [ 6.9344e-03, -3.2302e-03,  5.0286e-03,  ..., -7.6372e-03,
          3.3706e-04,  3.2491e-04],
        [ 1.6321e-02, -6.3958e-03,  5.0811e-03,  ..., -4.7124e-03,
          3.7874e-03,  3.6046e-03],
        [-9.5407e-04, -9.5503e-03,  1.0742e-02,  ..., -4.9108e-03,
          6.3167e-03,  5.7234e-03]], device='cuda:0'), 'base_model.model.decoder.block.2.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0043,  0.0264, -0.0115,  ...,  0.0311,  0.0349,  0.0248],
        [-0.0181, -0.0033,  0.0367,  ...,  0.0160, -0.0377, -0.0214],
        [ 0.0447, -0.0075, -0.0205,  ...,  0.0276,  0.0195, -0.0050],
        ...,
        [-0.0450, -0.0030,  0.0271,  ..., -0.0239,  0.0129,  0.0078],
        [-0.0393, -0.0002,  0.0071,  ..., -0.0083,  0.0064, -0.0282],
        [ 0.0125, -0.0222,  0.0274,  ..., -0.0385, -0.0269, -0.0489]],
       device='cuda:0'), 'base_model.model.decoder.block.2.layer.0.SelfAttention.q.lora_B.weight': tensor([[-3.1170e-03,  1.0720e-02,  2.1632e-03,  ..., -8.6142e-04,
         -1.2131e-03,  3.1180e-03],
        [ 6.1501e-03, -1.2021e-02,  3.3474e-07,  ..., -8.4009e-03,
         -3.4500e-03, -8.8669e-03],
        [-7.0843e-03,  5.7099e-04, -1.1848e-02,  ...,  2.5516e-03,
          5.1409e-03,  1.3616e-03],
        ...,
        [ 2.2997e-03,  5.7992e-03, -2.0466e-02,  ...,  1.0433e-02,
          1.6501e-02, -2.6774e-03],
        [-6.2623e-03,  4.4403e-04, -1.5141e-02,  ...,  6.5718e-03,
          1.8738e-02,  4.3951e-03],
        [-4.7032e-03,  5.6032e-03, -1.0028e-02,  ...,  9.1350e-03,
          8.2068e-03,  4.3547e-03]], device='cuda:0'), 'base_model.model.decoder.block.2.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0162, -0.0015,  0.0074,  ..., -0.0003,  0.0158, -0.0235],
        [-0.0018, -0.0206,  0.0025,  ...,  0.0132, -0.0139,  0.0327],
        [ 0.0019, -0.0290, -0.0073,  ..., -0.0219,  0.0109,  0.0003],
        ...,
        [ 0.0291, -0.0131, -0.0077,  ..., -0.0039, -0.0046, -0.0121],
        [ 0.0109, -0.0214,  0.0038,  ...,  0.0181, -0.0220, -0.0040],
        [-0.0069, -0.0031,  0.0367,  ..., -0.0086,  0.0165,  0.0042]],
       device='cuda:0'), 'base_model.model.decoder.block.2.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 2.4285e-02,  1.9491e-02,  2.0244e-02,  ...,  2.0564e-02,
         -2.1249e-02,  2.2550e-02],
        [ 1.6146e-02,  2.1073e-02,  2.0157e-02,  ...,  2.2195e-02,
         -1.9886e-02,  1.3432e-02],
        [-2.4051e-03, -3.1521e-03, -2.6053e-03,  ..., -8.9358e-03,
          2.8409e-03, -6.3286e-03],
        ...,
        [ 8.3711e-05, -5.0593e-03,  6.4664e-04,  ..., -3.6170e-03,
         -2.2688e-05, -3.6618e-03],
        [ 2.6448e-02,  2.5704e-02,  2.2131e-02,  ...,  2.7836e-02,
         -2.6099e-02,  1.0297e-02],
        [-2.4451e-02, -2.4363e-02, -2.2445e-02,  ..., -2.3449e-02,
          2.3975e-02, -2.5908e-02]], device='cuda:0'), 'base_model.model.decoder.block.2.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-3.2235e-02, -2.4453e-02, -1.8897e-03,  ..., -6.8799e-03,
         -4.4862e-02, -1.3606e-02],
        [ 2.0254e-02, -1.5936e-02,  4.4509e-03,  ...,  2.1429e-04,
         -1.0414e-02,  9.5917e-03],
        [-3.7509e-02, -7.7726e-03,  8.3584e-03,  ...,  8.5235e-05,
         -4.5792e-02,  1.7381e-02],
        ...,
        [ 1.3659e-02, -3.2639e-02,  1.3874e-02,  ..., -9.9682e-03,
          7.1122e-03, -1.3168e-02],
        [ 2.7758e-02,  1.7779e-02,  2.0833e-02,  ...,  9.7087e-03,
          4.2300e-02,  1.4540e-02],
        [-4.2461e-03,  3.1331e-03,  2.5784e-02,  ..., -3.8201e-02,
          6.7687e-03, -1.6261e-02]], device='cuda:0'), 'base_model.model.decoder.block.2.layer.1.EncDecAttention.q.lora_B.weight': tensor([[-0.0192, -0.0265, -0.0246,  ...,  0.0291,  0.0167, -0.0226],
        [ 0.0060,  0.0069,  0.0018,  ..., -0.0137, -0.0044,  0.0027],
        [ 0.0109,  0.0068,  0.0197,  ..., -0.0124, -0.0072,  0.0091],
        ...,
        [ 0.0187, -0.0130,  0.0149,  ..., -0.0143, -0.0120,  0.0074],
        [-0.0028,  0.0222,  0.0006,  ..., -0.0003,  0.0002,  0.0081],
        [-0.0265, -0.0019, -0.0296,  ...,  0.0252,  0.0172, -0.0095]],
       device='cuda:0'), 'base_model.model.decoder.block.2.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0110, -0.0249, -0.0031,  ..., -0.0298,  0.0260,  0.0516],
        [-0.0553,  0.0024,  0.0013,  ..., -0.0013,  0.0018,  0.0188],
        [-0.0096, -0.0039, -0.0053,  ..., -0.0048,  0.0163,  0.0174],
        ...,
        [-0.0267, -0.0287,  0.0545,  ...,  0.0190,  0.0132,  0.0087],
        [-0.0091,  0.0195,  0.0129,  ..., -0.0401,  0.0185,  0.0505],
        [ 0.0057, -0.0250,  0.0268,  ..., -0.0211,  0.0299,  0.0019]],
       device='cuda:0'), 'base_model.model.decoder.block.2.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 1.3512e-02,  1.0094e-02,  1.0749e-02,  ...,  1.3937e-02,
          1.4577e-02,  1.3163e-02],
        [-1.6808e-02, -1.2401e-02, -2.2549e-02,  ..., -1.5579e-02,
         -1.5503e-02, -1.6389e-02],
        [-6.7631e-03, -4.8791e-03, -9.9732e-03,  ..., -8.7470e-03,
         -7.3061e-03, -9.7309e-03],
        ...,
        [ 7.1581e-05, -2.2548e-03, -8.9037e-03,  ..., -6.1649e-03,
         -4.6464e-03, -6.3443e-03],
        [-2.1520e-02, -1.6354e-02, -2.1507e-02,  ..., -1.8473e-02,
         -2.1734e-02, -2.0071e-02],
        [-2.2858e-04, -1.2750e-03,  5.7743e-04,  ...,  1.1762e-03,
          2.3638e-03,  2.9686e-03]], device='cuda:0'), 'base_model.model.decoder.block.3.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0072, -0.0215,  0.0113,  ..., -0.0083, -0.0112, -0.0261],
        [ 0.0192,  0.0184,  0.0049,  ..., -0.0295,  0.0275, -0.0116],
        [ 0.0280,  0.0047, -0.0025,  ...,  0.0011, -0.0008,  0.0254],
        ...,
        [-0.0189,  0.0099, -0.0184,  ..., -0.0014,  0.0009,  0.0082],
        [-0.0026, -0.0022,  0.0208,  ..., -0.0045,  0.0279, -0.0247],
        [ 0.0204, -0.0134, -0.0076,  ...,  0.0101, -0.0260,  0.0311]],
       device='cuda:0'), 'base_model.model.decoder.block.3.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0130,  0.0056,  0.0038,  ..., -0.0028, -0.0109,  0.0039],
        [ 0.0065, -0.0119,  0.0148,  ..., -0.0166,  0.0059,  0.0121],
        [-0.0192,  0.0150,  0.0005,  ...,  0.0006, -0.0156,  0.0008],
        ...,
        [-0.0164, -0.0180,  0.0115,  ..., -0.0031, -0.0211,  0.0231],
        [ 0.0160,  0.0132, -0.0068,  ...,  0.0005,  0.0191, -0.0242],
        [ 0.0048, -0.0069,  0.0201,  ..., -0.0211, -0.0003,  0.0144]],
       device='cuda:0'), 'base_model.model.decoder.block.3.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0154, -0.0190, -0.0139,  ..., -0.0068,  0.0060, -0.0097],
        [-0.0084,  0.0118, -0.0295,  ..., -0.0322, -0.0392, -0.0278],
        [ 0.0255, -0.0257,  0.0247,  ...,  0.0118, -0.0110,  0.0428],
        ...,
        [ 0.0037,  0.0066,  0.0163,  ...,  0.0142, -0.0087, -0.0071],
        [ 0.0321,  0.0076,  0.0286,  ..., -0.0035,  0.0288,  0.0387],
        [-0.0373, -0.0299, -0.0162,  ..., -0.0288, -0.0450, -0.0064]],
       device='cuda:0'), 'base_model.model.decoder.block.3.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0055,  0.0059, -0.0068,  ...,  0.0066, -0.0059,  0.0058],
        [ 0.0017,  0.0039, -0.0019,  ...,  0.0005,  0.0024, -0.0008],
        [-0.0046, -0.0104,  0.0097,  ..., -0.0098,  0.0082, -0.0097],
        ...,
        [-0.0010,  0.0006,  0.0019,  ...,  0.0017, -0.0012, -0.0005],
        [ 0.0027,  0.0070, -0.0015,  ...,  0.0019, -0.0003,  0.0019],
        [-0.0111, -0.0121,  0.0130,  ..., -0.0129,  0.0124, -0.0135]],
       device='cuda:0'), 'base_model.model.decoder.block.3.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0305,  0.0111,  0.0111,  ...,  0.0119, -0.0292, -0.0201],
        [ 0.0215,  0.0097, -0.0449,  ..., -0.0125,  0.0371,  0.0259],
        [-0.0006, -0.0228,  0.0173,  ..., -0.0298, -0.0348, -0.0166],
        ...,
        [-0.0496, -0.0060,  0.0336,  ..., -0.0376, -0.0366,  0.0158],
        [ 0.0150, -0.0011, -0.0006,  ...,  0.0357,  0.0258,  0.0381],
        [-0.0085, -0.0071,  0.0198,  ...,  0.0362, -0.0071,  0.0058]],
       device='cuda:0'), 'base_model.model.decoder.block.3.layer.1.EncDecAttention.q.lora_B.weight': tensor([[-0.0022, -0.0042,  0.0066,  ...,  0.0047, -0.0036, -0.0020],
        [ 0.0189, -0.0371,  0.0382,  ...,  0.0305, -0.0319, -0.0206],
        [-0.0052, -0.0098,  0.0078,  ...,  0.0078, -0.0058, -0.0184],
        ...,
        [ 0.0121,  0.0118, -0.0107,  ..., -0.0072,  0.0152, -0.0095],
        [-0.0156,  0.0325, -0.0318,  ..., -0.0103,  0.0305,  0.0194],
        [ 0.0130,  0.0154, -0.0188,  ..., -0.0170,  0.0142,  0.0074]],
       device='cuda:0'), 'base_model.model.decoder.block.3.layer.1.EncDecAttention.v.lora_A.weight': tensor([[ 0.0188, -0.0262,  0.0216,  ...,  0.0069, -0.0215, -0.0048],
        [ 0.0322, -0.0194,  0.0094,  ...,  0.0157, -0.0021,  0.0539],
        [-0.0159,  0.0084,  0.0091,  ..., -0.0113,  0.0037,  0.0314],
        ...,
        [ 0.0004, -0.0198,  0.0015,  ..., -0.0551,  0.0388,  0.0306],
        [ 0.0350, -0.0306, -0.0190,  ...,  0.0378, -0.0301, -0.0262],
        [ 0.0203,  0.0142, -0.0191,  ..., -0.0139,  0.0464,  0.0181]],
       device='cuda:0'), 'base_model.model.decoder.block.3.layer.1.EncDecAttention.v.lora_B.weight': tensor([[-0.0019, -0.0041,  0.0024,  ..., -0.0003, -0.0006,  0.0029],
        [-0.0188,  0.0186,  0.0175,  ...,  0.0169, -0.0168,  0.0166],
        [ 0.0126, -0.0167, -0.0079,  ..., -0.0090,  0.0082, -0.0114],
        ...,
        [-0.0065,  0.0139,  0.0076,  ...,  0.0082, -0.0083,  0.0121],
        [-0.0111,  0.0149,  0.0127,  ...,  0.0131, -0.0134,  0.0118],
        [ 0.0017,  0.0036, -0.0023,  ..., -0.0028,  0.0031, -0.0028]],
       device='cuda:0'), 'base_model.model.decoder.block.4.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0342, -0.0317, -0.0014,  ..., -0.0271,  0.0120,  0.0191],
        [-0.0287, -0.0342,  0.0428,  ...,  0.0099, -0.0305,  0.0277],
        [-0.0308, -0.0273, -0.0005,  ..., -0.0049, -0.0236, -0.0056],
        ...,
        [-0.0138, -0.0167, -0.0405,  ...,  0.0315,  0.0343,  0.0241],
        [-0.0294, -0.0314,  0.0403,  ...,  0.0076, -0.0130, -0.0139],
        [-0.0245, -0.0102,  0.0352,  ..., -0.0369, -0.0388,  0.0226]],
       device='cuda:0'), 'base_model.model.decoder.block.4.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0039,  0.0033,  0.0120,  ...,  0.0007, -0.0020,  0.0129],
        [ 0.0154,  0.0150,  0.0096,  ..., -0.0125,  0.0154,  0.0157],
        [ 0.0114,  0.0126,  0.0129,  ..., -0.0122,  0.0115,  0.0021],
        ...,
        [-0.0076, -0.0078, -0.0041,  ..., -0.0005, -0.0058, -0.0095],
        [ 0.0091,  0.0096,  0.0089,  ..., -0.0087,  0.0105,  0.0073],
        [-0.0055, -0.0083, -0.0074,  ...,  0.0106, -0.0107, -0.0063]],
       device='cuda:0'), 'base_model.model.decoder.block.4.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0116,  0.0151, -0.0338,  ...,  0.0024,  0.0069,  0.0019],
        [-0.0075,  0.0279, -0.0158,  ..., -0.0128,  0.0325,  0.0015],
        [ 0.0325, -0.0339,  0.0060,  ..., -0.0077,  0.0005,  0.0309],
        ...,
        [ 0.0232,  0.0226, -0.0187,  ...,  0.0152, -0.0232,  0.0162],
        [ 0.0283,  0.0220,  0.0349,  ...,  0.0378,  0.0139,  0.0358],
        [ 0.0147,  0.0302, -0.0180,  ...,  0.0072, -0.0060, -0.0410]],
       device='cuda:0'), 'base_model.model.decoder.block.4.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0115,  0.0098,  0.0073,  ...,  0.0100,  0.0085, -0.0100],
        [ 0.0131, -0.0123, -0.0133,  ..., -0.0141, -0.0025,  0.0153],
        [-0.0161,  0.0144,  0.0136,  ...,  0.0137,  0.0108, -0.0134],
        ...,
        [-0.0047,  0.0031,  0.0027,  ...,  0.0071,  0.0149, -0.0057],
        [ 0.0156, -0.0158, -0.0076,  ..., -0.0147, -0.0157,  0.0160],
        [-0.0158,  0.0169,  0.0219,  ...,  0.0165,  0.0194, -0.0162]],
       device='cuda:0'), 'base_model.model.decoder.block.4.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0263,  0.0215,  0.0172,  ..., -0.0139, -0.0270, -0.0057],
        [ 0.0239, -0.0136,  0.0008,  ...,  0.0056,  0.0047, -0.0103],
        [ 0.0140,  0.0015, -0.0131,  ...,  0.0060,  0.0066,  0.0215],
        ...,
        [ 0.0302, -0.0049, -0.0103,  ...,  0.0099, -0.0047, -0.0163],
        [ 0.0014,  0.0275,  0.0259,  ...,  0.0201, -0.0150,  0.0098],
        [ 0.0081,  0.0232, -0.0162,  ..., -0.0027, -0.0194,  0.0108]],
       device='cuda:0'), 'base_model.model.decoder.block.4.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0242,  0.0186, -0.0200,  ...,  0.0223, -0.0155,  0.0179],
        [-0.0203, -0.0142,  0.0140,  ..., -0.0150,  0.0092, -0.0160],
        [-0.0007,  0.0006, -0.0005,  ...,  0.0006, -0.0002, -0.0026],
        ...,
        [ 0.0092,  0.0041, -0.0047,  ...,  0.0082, -0.0055,  0.0086],
        [ 0.0151,  0.0137, -0.0022,  ...,  0.0107, -0.0017,  0.0091],
        [-0.0118, -0.0104,  0.0048,  ..., -0.0077, -0.0013, -0.0059]],
       device='cuda:0'), 'base_model.model.decoder.block.4.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0023, -0.0160,  0.0328,  ...,  0.0381,  0.0126, -0.0562],
        [-0.0314,  0.0148,  0.0142,  ...,  0.0375, -0.0140, -0.0295],
        [-0.0111,  0.0272,  0.0154,  ...,  0.0156,  0.0067, -0.0610],
        ...,
        [-0.0016, -0.0208,  0.0008,  ...,  0.0163,  0.0187,  0.0313],
        [ 0.0035, -0.0200,  0.0226,  ..., -0.0011, -0.0050,  0.0417],
        [ 0.0164,  0.0416, -0.0165,  ..., -0.0197,  0.0283, -0.0016]],
       device='cuda:0'), 'base_model.model.decoder.block.4.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 0.0080,  0.0172,  0.0179,  ...,  0.0031, -0.0199, -0.0094],
        [ 0.0060,  0.0121,  0.0121,  ..., -0.0026, -0.0113, -0.0111],
        [-0.0091, -0.0273, -0.0263,  ...,  0.0116,  0.0272,  0.0177],
        ...,
        [-0.0019, -0.0016,  0.0025,  ...,  0.0041,  0.0043,  0.0018],
        [-0.0042, -0.0017, -0.0011,  ..., -0.0032, -0.0010,  0.0023],
        [ 0.0004,  0.0092,  0.0103,  ..., -0.0058, -0.0173, -0.0072]],
       device='cuda:0'), 'base_model.model.decoder.block.5.layer.0.SelfAttention.q.lora_A.weight': tensor([[-2.6730e-02,  2.0864e-02,  2.3388e-02,  ..., -1.5293e-02,
         -1.6101e-03, -4.0400e-02],
        [-3.2931e-02, -1.0590e-03,  1.2445e-02,  ..., -5.5235e-03,
          3.3820e-02, -2.0991e-02],
        [ 1.2719e-02,  2.5506e-03, -3.6293e-02,  ...,  1.6096e-02,
          1.3012e-02,  1.6014e-02],
        ...,
        [-1.3189e-02, -2.4657e-02, -8.4241e-03,  ..., -5.5609e-03,
         -4.4067e-03, -3.3457e-02],
        [ 2.7041e-02,  2.7457e-02, -2.4774e-03,  ...,  2.2314e-03,
          1.1362e-02, -9.3929e-03],
        [ 1.5837e-02, -9.2169e-03, -7.3251e-04,  ..., -2.6933e-05,
          1.2411e-02, -4.1066e-03]], device='cuda:0'), 'base_model.model.decoder.block.5.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0014,  0.0082,  0.0049,  ..., -0.0001, -0.0062,  0.0002],
        [-0.0049, -0.0086, -0.0027,  ..., -0.0024,  0.0020,  0.0039],
        [-0.0048, -0.0091, -0.0022,  ..., -0.0030,  0.0021,  0.0041],
        ...,
        [-0.0004,  0.0162,  0.0006,  ...,  0.0005, -0.0079,  0.0013],
        [-0.0036, -0.0084,  0.0063,  ..., -0.0062,  0.0043,  0.0085],
        [-0.0044,  0.0027,  0.0058,  ..., -0.0065, -0.0014,  0.0068]],
       device='cuda:0'), 'base_model.model.decoder.block.5.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0186, -0.0075,  0.0282,  ..., -0.0128, -0.0348, -0.0162],
        [ 0.0120,  0.0150, -0.0364,  ..., -0.0095,  0.0025, -0.0103],
        [ 0.0250,  0.0160, -0.0067,  ..., -0.0135,  0.0271,  0.0212],
        ...,
        [ 0.0176, -0.0096,  0.0075,  ...,  0.0166, -0.0108, -0.0156],
        [ 0.0347,  0.0249, -0.0111,  ..., -0.0084,  0.0292,  0.0088],
        [-0.0105,  0.0333,  0.0109,  ..., -0.0365,  0.0088, -0.0124]],
       device='cuda:0'), 'base_model.model.decoder.block.5.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0106,  0.0103,  0.0088,  ...,  0.0104,  0.0139, -0.0114],
        [ 0.0053, -0.0025, -0.0100,  ..., -0.0028, -0.0017,  0.0061],
        [ 0.0084, -0.0051, -0.0125,  ..., -0.0073, -0.0063,  0.0097],
        ...,
        [ 0.0039,  0.0021, -0.0035,  ..., -0.0035, -0.0005,  0.0048],
        [-0.0107,  0.0107,  0.0084,  ...,  0.0114,  0.0113, -0.0080],
        [ 0.0057, -0.0105, -0.0100,  ..., -0.0036, -0.0060,  0.0014]],
       device='cuda:0'), 'base_model.model.decoder.block.5.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0253,  0.0348,  0.0144,  ...,  0.0535, -0.0086, -0.0021],
        [ 0.0331, -0.0260,  0.0130,  ..., -0.0028, -0.0292,  0.0152],
        [ 0.0248, -0.0087, -0.0264,  ..., -0.0248, -0.0025,  0.0110],
        ...,
        [ 0.0223, -0.0317,  0.0493,  ..., -0.0156, -0.0037, -0.0297],
        [ 0.0025,  0.0178, -0.0461,  ..., -0.0236,  0.0002,  0.0295],
        [ 0.0259, -0.0477,  0.0526,  ..., -0.0363, -0.0160, -0.0426]],
       device='cuda:0'), 'base_model.model.decoder.block.5.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0095, -0.0096, -0.0092,  ..., -0.0071,  0.0069, -0.0087],
        [ 0.0015, -0.0024, -0.0026,  ..., -0.0026,  0.0081, -0.0055],
        [-0.0016,  0.0003,  0.0006,  ...,  0.0035, -0.0067, -0.0027],
        ...,
        [ 0.0003,  0.0031, -0.0029,  ...,  0.0003,  0.0018, -0.0005],
        [-0.0036,  0.0015,  0.0029,  ...,  0.0102, -0.0047,  0.0026],
        [-0.0088,  0.0082,  0.0067,  ..., -0.0148,  0.0157, -0.0170]],
       device='cuda:0'), 'base_model.model.decoder.block.5.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0108, -0.0207, -0.0201,  ..., -0.0062, -0.0595, -0.0536],
        [ 0.0466,  0.0231,  0.0192,  ...,  0.0111,  0.0117,  0.0167],
        [ 0.0248, -0.0007, -0.0118,  ..., -0.0208,  0.0139,  0.0163],
        ...,
        [ 0.0050, -0.0228,  0.0266,  ...,  0.0138,  0.0375,  0.0285],
        [ 0.0072,  0.0144,  0.0285,  ...,  0.0107,  0.0474,  0.0207],
        [ 0.0098,  0.0003, -0.0225,  ..., -0.0130,  0.0355,  0.0492]],
       device='cuda:0'), 'base_model.model.decoder.block.5.layer.1.EncDecAttention.v.lora_B.weight': tensor([[-2.1625e-03,  1.6814e-04,  8.1146e-05,  ...,  1.4400e-03,
         -5.4853e-03, -1.1964e-03],
        [ 6.4509e-03, -6.9242e-03, -1.6537e-02,  ..., -6.6245e-03,
         -2.1439e-02, -1.5015e-02],
        [-8.9251e-04,  7.1215e-04, -2.7612e-04,  ...,  2.6042e-04,
          1.3185e-02,  9.8170e-03],
        ...,
        [-3.6989e-03,  1.8064e-03,  8.6825e-03,  ...,  2.5273e-03,
          4.6486e-03,  5.4987e-03],
        [-1.1506e-02,  8.4292e-03,  5.8922e-03,  ...,  1.1264e-02,
          4.8699e-03,  7.4301e-03],
        [ 1.0491e-02, -9.8668e-03, -1.0596e-02,  ..., -9.7787e-03,
         -5.6693e-03, -9.9868e-03]], device='cuda:0'), 'base_model.model.decoder.block.6.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0067, -0.0038,  0.0387,  ...,  0.0008, -0.0110,  0.0104],
        [ 0.0157, -0.0423,  0.0041,  ..., -0.0276,  0.0129,  0.0129],
        [-0.0351,  0.0106, -0.0120,  ...,  0.0130, -0.0018, -0.0170],
        ...,
        [ 0.0046,  0.0349,  0.0031,  ...,  0.0075,  0.0278,  0.0109],
        [ 0.0355,  0.0083,  0.0434,  ..., -0.0034, -0.0004, -0.0221],
        [-0.0067,  0.0074,  0.0048,  ...,  0.0083, -0.0431, -0.0202]],
       device='cuda:0'), 'base_model.model.decoder.block.6.layer.0.SelfAttention.q.lora_B.weight': tensor([[-1.4334e-02, -2.0122e-02,  2.5548e-02,  ...,  1.8705e-02,
          4.1224e-04, -9.1952e-03],
        [-3.4477e-03, -6.8384e-04, -4.9736e-05,  ...,  2.7096e-03,
          9.1590e-03,  1.4286e-02],
        [-1.1760e-02, -1.6113e-02,  1.9543e-02,  ...,  1.4789e-02,
          3.0643e-03, -6.1182e-03],
        ...,
        [ 1.0866e-02,  1.0069e-02, -8.9908e-03,  ..., -9.9777e-03,
          1.0297e-02,  7.7944e-03],
        [-1.1795e-02, -1.1093e-02,  1.2621e-02,  ...,  1.2965e-02,
         -9.7311e-03, -1.1893e-02],
        [ 1.2635e-02,  1.1360e-02, -1.1950e-02,  ..., -1.2414e-02,
          1.0531e-02,  1.4034e-02]], device='cuda:0'), 'base_model.model.decoder.block.6.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0033, -0.0322,  0.0411,  ..., -0.0003, -0.0140,  0.0132],
        [ 0.0124, -0.0387,  0.0131,  ..., -0.0345,  0.0151,  0.0003],
        [-0.0095,  0.0191, -0.0311,  ..., -0.0108, -0.0286, -0.0093],
        ...,
        [-0.0186,  0.0239, -0.0576,  ..., -0.0223, -0.0066, -0.0108],
        [ 0.0070, -0.0496, -0.0148,  ..., -0.0102, -0.0159,  0.0199],
        [ 0.0071,  0.0019, -0.0213,  ...,  0.0124, -0.0183,  0.0216]],
       device='cuda:0'), 'base_model.model.decoder.block.6.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 2.4060e-02,  2.1683e-02, -2.5614e-02,  ..., -2.4639e-02,
          2.5046e-02,  2.4352e-02],
        [ 1.3227e-02,  9.1044e-03, -1.3184e-02,  ..., -1.1382e-02,
          1.2999e-02,  1.3374e-02],
        [-1.4519e-02, -1.2697e-02,  1.2229e-02,  ...,  1.1532e-02,
         -1.4862e-02, -1.5476e-02],
        ...,
        [-1.6640e-02, -1.5588e-02,  1.2420e-02,  ...,  2.0494e-02,
         -1.3000e-02, -1.1540e-02],
        [ 1.6104e-02,  1.6665e-02, -8.1493e-03,  ..., -2.1218e-02,
          8.9385e-03,  8.0249e-03],
        [ 6.4023e-05,  1.2668e-03, -8.7618e-03,  ...,  3.2057e-05,
          8.1570e-03,  9.9918e-03]], device='cuda:0'), 'base_model.model.decoder.block.6.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0133,  0.0064,  0.0368,  ...,  0.0051, -0.0354, -0.0184],
        [-0.0097,  0.0049, -0.0109,  ..., -0.0311, -0.0086, -0.0254],
        [-0.0274,  0.0319, -0.0215,  ...,  0.0142,  0.0177,  0.0154],
        ...,
        [-0.0239, -0.0018,  0.0185,  ..., -0.0107, -0.0099,  0.0124],
        [-0.0030, -0.0054, -0.0009,  ..., -0.0291,  0.0086,  0.0105],
        [-0.0358, -0.0142, -0.0217,  ..., -0.0015,  0.0169,  0.0144]],
       device='cuda:0'), 'base_model.model.decoder.block.6.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0037,  0.0140,  0.0248,  ...,  0.0287, -0.0060,  0.0007],
        [-0.0074,  0.0020, -0.0056,  ...,  0.0008,  0.0032,  0.0039],
        [-0.0019, -0.0019,  0.0006,  ...,  0.0045, -0.0034,  0.0044],
        ...,
        [ 0.0046, -0.0029,  0.0052,  ...,  0.0034,  0.0060, -0.0008],
        [ 0.0097,  0.0037,  0.0133,  ...,  0.0158, -0.0098, -0.0095],
        [ 0.0096, -0.0046,  0.0003,  ..., -0.0082,  0.0195, -0.0122]],
       device='cuda:0'), 'base_model.model.decoder.block.6.layer.1.EncDecAttention.v.lora_A.weight': tensor([[ 0.0266, -0.0150, -0.0428,  ...,  0.0195,  0.0117,  0.0346],
        [ 0.0236, -0.0423,  0.0263,  ..., -0.0011, -0.0402, -0.0415],
        [-0.0090,  0.0466, -0.0165,  ..., -0.0530, -0.0045,  0.0174],
        ...,
        [-0.0144, -0.0242,  0.0267,  ...,  0.0429, -0.0238, -0.0078],
        [ 0.0131, -0.0153,  0.0209,  ...,  0.0367, -0.0131, -0.0385],
        [ 0.0002,  0.0393, -0.0139,  ..., -0.0045,  0.0177,  0.0329]],
       device='cuda:0'), 'base_model.model.decoder.block.6.layer.1.EncDecAttention.v.lora_B.weight': tensor([[-0.0009, -0.0140,  0.0066,  ..., -0.0093,  0.0028,  0.0081],
        [-0.0098,  0.0039, -0.0037,  ...,  0.0026,  0.0042, -0.0022],
        [ 0.0084,  0.0001,  0.0004,  ...,  0.0047, -0.0066,  0.0003],
        ...,
        [ 0.0044,  0.0213, -0.0220,  ...,  0.0198,  0.0182, -0.0189],
        [-0.0040,  0.0051, -0.0058,  ...,  0.0041,  0.0030, -0.0046],
        [-0.0051,  0.0010,  0.0045,  ..., -0.0046, -0.0037,  0.0040]],
       device='cuda:0'), 'base_model.model.decoder.block.7.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0331, -0.0111, -0.0323,  ...,  0.0251,  0.0505, -0.0046],
        [-0.0215,  0.0399,  0.0451,  ..., -0.0216, -0.0509, -0.0254],
        [-0.0085,  0.0070,  0.0028,  ..., -0.0233,  0.0098, -0.0006],
        ...,
        [-0.0136, -0.0158,  0.0197,  ..., -0.0471,  0.0237, -0.0038],
        [ 0.0197, -0.0128, -0.0333,  ...,  0.0287,  0.0450, -0.0127],
        [ 0.0028,  0.0185, -0.0092,  ..., -0.0441, -0.0428,  0.0232]],
       device='cuda:0'), 'base_model.model.decoder.block.7.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0008,  0.0002, -0.0002,  ..., -0.0006,  0.0024,  0.0010],
        [ 0.0004,  0.0005, -0.0004,  ..., -0.0005,  0.0018,  0.0012],
        [ 0.0012, -0.0009,  0.0010,  ..., -0.0017,  0.0024, -0.0002],
        ...,
        [ 0.0071, -0.0029, -0.0035,  ..., -0.0041,  0.0080, -0.0040],
        [ 0.0006, -0.0016,  0.0108,  ...,  0.0008, -0.0043, -0.0014],
        [ 0.0016,  0.0023,  0.0044,  ...,  0.0002,  0.0046,  0.0030]],
       device='cuda:0'), 'base_model.model.decoder.block.7.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0091, -0.0108, -0.0018,  ...,  0.0179,  0.0270,  0.0174],
        [-0.0182,  0.0060, -0.0365,  ...,  0.0066, -0.0001, -0.0285],
        [-0.0408,  0.0361, -0.0176,  ...,  0.0240,  0.0034,  0.0067],
        ...,
        [ 0.0293,  0.0057,  0.0186,  ...,  0.0305, -0.0190,  0.0132],
        [-0.0201, -0.0120,  0.0187,  ..., -0.0276, -0.0281, -0.0402],
        [ 0.0082, -0.0213,  0.0019,  ...,  0.0298, -0.0016, -0.0259]],
       device='cuda:0'), 'base_model.model.decoder.block.7.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0195, -0.0055, -0.0053,  ...,  0.0047, -0.0101,  0.0149],
        [ 0.0014, -0.0081, -0.0072,  ...,  0.0070, -0.0058, -0.0003],
        [ 0.0088,  0.0007,  0.0048,  ..., -0.0033,  0.0032,  0.0156],
        ...,
        [ 0.0045, -0.0026, -0.0015,  ...,  0.0017, -0.0012,  0.0068],
        [ 0.0081, -0.0016, -0.0009,  ...,  0.0013, -0.0054,  0.0117],
        [ 0.0039, -0.0103, -0.0033,  ...,  0.0092, -0.0028,  0.0041]],
       device='cuda:0'), 'base_model.model.decoder.block.7.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0203,  0.0043,  0.0081,  ...,  0.0358,  0.0223,  0.0172],
        [-0.0086, -0.0177,  0.0020,  ..., -0.0164, -0.0001,  0.0288],
        [-0.0135,  0.0085,  0.0117,  ...,  0.0381,  0.0329,  0.0229],
        ...,
        [-0.0364, -0.0218, -0.0393,  ...,  0.0238,  0.0459,  0.0361],
        [ 0.0031, -0.0134, -0.0096,  ...,  0.0044, -0.0182, -0.0195],
        [ 0.0105,  0.0036, -0.0233,  ...,  0.0360,  0.0076,  0.0006]],
       device='cuda:0'), 'base_model.model.decoder.block.7.layer.1.EncDecAttention.q.lora_B.weight': tensor([[-0.0246, -0.0249,  0.0212,  ..., -0.0247,  0.0232, -0.0254],
        [-0.0233, -0.0259,  0.0089,  ..., -0.0237,  0.0226, -0.0243],
        [-0.0282, -0.0282,  0.0009,  ..., -0.0280,  0.0279, -0.0301],
        ...,
        [-0.0124, -0.0138,  0.0094,  ..., -0.0132,  0.0121, -0.0126],
        [ 0.0109,  0.0019, -0.0137,  ...,  0.0081, -0.0119,  0.0121],
        [ 0.0128,  0.0076,  0.0082,  ...,  0.0131, -0.0137,  0.0123]],
       device='cuda:0'), 'base_model.model.decoder.block.7.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0427,  0.0195,  0.0449,  ...,  0.0135, -0.0402, -0.0507],
        [ 0.0125,  0.0196, -0.0151,  ..., -0.0420,  0.0035,  0.0245],
        [ 0.0263, -0.0365,  0.0320,  ...,  0.0032, -0.0379,  0.0087],
        ...,
        [ 0.0331,  0.0281, -0.0341,  ...,  0.0250,  0.0080,  0.0225],
        [-0.0033, -0.0399, -0.0354,  ...,  0.0027,  0.0030,  0.0226],
        [-0.0005,  0.0128,  0.0210,  ..., -0.0145,  0.0167, -0.0309]],
       device='cuda:0'), 'base_model.model.decoder.block.7.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 8.7193e-03, -9.4752e-03,  4.8379e-03,  ..., -6.9801e-03,
          4.6064e-03,  1.4173e-02],
        [ 1.0323e-02, -7.7941e-03,  1.0070e-02,  ..., -2.5394e-03,
          7.4860e-03, -1.9913e-03],
        [ 2.3713e-03, -3.8002e-03,  6.1123e-03,  ..., -6.1484e-03,
          8.3422e-03, -1.3110e-02],
        ...,
        [ 2.4322e-03,  1.1196e-02, -8.1483e-03,  ..., -9.3193e-03,
         -4.1116e-03,  6.0736e-03],
        [-4.6512e-03,  2.2509e-05,  2.8143e-03,  ...,  7.7112e-03,
          1.1012e-02, -1.5901e-02],
        [-2.5353e-02,  1.9855e-02, -1.2999e-02,  ...,  2.1589e-02,
          3.5988e-03, -1.5793e-02]], device='cuda:0'), 'base_model.model.decoder.block.8.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0055,  0.0482, -0.0049,  ...,  0.0258,  0.0268,  0.0282],
        [ 0.0004,  0.0061,  0.0378,  ..., -0.0061,  0.0131, -0.0106],
        [-0.0127, -0.0368,  0.0114,  ..., -0.0245, -0.0108,  0.0082],
        ...,
        [ 0.0270, -0.0469,  0.0392,  ..., -0.0057,  0.0246, -0.0449],
        [ 0.0149, -0.0116,  0.0343,  ...,  0.0086, -0.0134, -0.0224],
        [-0.0031, -0.0186,  0.0189,  ...,  0.0102,  0.0251,  0.0056]],
       device='cuda:0'), 'base_model.model.decoder.block.8.layer.0.SelfAttention.q.lora_B.weight': tensor([[-8.1862e-03,  1.1881e-02,  9.7900e-03,  ...,  1.0970e-02,
         -4.8307e-03,  1.3177e-02],
        [ 1.3475e-02, -1.3164e-02, -1.6153e-02,  ..., -3.9508e-03,
         -5.1986e-05, -6.6920e-03],
        [ 1.3827e-03,  3.2577e-03, -4.9111e-03,  ...,  5.6041e-03,
          3.4114e-03, -1.3535e-03],
        ...,
        [-2.6834e-02,  2.3956e-02,  2.7925e-02,  ...,  2.6604e-02,
          2.5881e-02, -4.2479e-03],
        [ 9.8006e-03,  5.8498e-03, -1.5935e-02,  ..., -1.2355e-02,
         -2.2700e-03,  1.5965e-02],
        [ 1.9455e-02, -1.4621e-02, -1.9711e-02,  ..., -2.1926e-02,
         -1.4545e-02, -2.0314e-03]], device='cuda:0'), 'base_model.model.decoder.block.8.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0086, -0.0127,  0.0164,  ..., -0.0085,  0.0160,  0.0239],
        [-0.0253,  0.0110, -0.0184,  ...,  0.0062, -0.0089,  0.0040],
        [ 0.0300, -0.0046, -0.0077,  ...,  0.0109,  0.0151,  0.0012],
        ...,
        [ 0.0069, -0.0402,  0.0182,  ..., -0.0243,  0.0367,  0.0397],
        [ 0.0392, -0.0008,  0.0079,  ..., -0.0269, -0.0036, -0.0002],
        [ 0.0118, -0.0075, -0.0023,  ..., -0.0224, -0.0161,  0.0017]],
       device='cuda:0'), 'base_model.model.decoder.block.8.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0047,  0.0029, -0.0069,  ..., -0.0018, -0.0063, -0.0014],
        [ 0.0082, -0.0077,  0.0052,  ...,  0.0103,  0.0080,  0.0121],
        [-0.0195,  0.0196, -0.0149,  ..., -0.0190, -0.0054, -0.0194],
        ...,
        [-0.0066,  0.0018, -0.0003,  ..., -0.0015, -0.0070,  0.0014],
        [ 0.0169, -0.0169,  0.0163,  ...,  0.0180, -0.0044,  0.0176],
        [ 0.0115, -0.0148,  0.0143,  ...,  0.0089,  0.0165,  0.0110]],
       device='cuda:0'), 'base_model.model.decoder.block.8.layer.1.EncDecAttention.q.lora_A.weight': tensor([[ 0.0088, -0.0198,  0.0352,  ...,  0.0017, -0.0137,  0.0235],
        [ 0.0074,  0.0117,  0.0034,  ..., -0.0414, -0.0173, -0.0157],
        [-0.0269,  0.0180,  0.0023,  ..., -0.0271, -0.0041, -0.0395],
        ...,
        [-0.0052, -0.0061, -0.0074,  ..., -0.0023,  0.0183,  0.0248],
        [-0.0085,  0.0143,  0.0291,  ..., -0.0317,  0.0161, -0.0440],
        [-0.0242,  0.0251,  0.0268,  ..., -0.0222,  0.0009,  0.0223]],
       device='cuda:0'), 'base_model.model.decoder.block.8.layer.1.EncDecAttention.q.lora_B.weight': tensor([[-0.0070, -0.0066, -0.0060,  ...,  0.0057, -0.0022, -0.0063],
        [ 0.0075,  0.0020,  0.0047,  ..., -0.0082, -0.0012,  0.0090],
        [-0.0149, -0.0074, -0.0133,  ...,  0.0114,  0.0035, -0.0127],
        ...,
        [-0.0060, -0.0045, -0.0088,  ...,  0.0093, -0.0050, -0.0093],
        [-0.0044,  0.0054, -0.0040,  ...,  0.0045,  0.0142, -0.0048],
        [ 0.0050,  0.0038,  0.0082,  ..., -0.0098, -0.0018,  0.0068]],
       device='cuda:0'), 'base_model.model.decoder.block.8.layer.1.EncDecAttention.v.lora_A.weight': tensor([[ 0.0147,  0.0348, -0.0170,  ..., -0.0441,  0.0298,  0.0520],
        [-0.0203,  0.0083,  0.0499,  ...,  0.0094,  0.0094, -0.0457],
        [ 0.0123,  0.0063,  0.0027,  ...,  0.0321, -0.0489, -0.0257],
        ...,
        [ 0.0059,  0.0268, -0.0280,  ...,  0.0138, -0.0344, -0.0239],
        [ 0.0025, -0.0047, -0.0095,  ...,  0.0065, -0.0235,  0.0059],
        [-0.0239, -0.0193, -0.0089,  ..., -0.0353, -0.0026,  0.0523]],
       device='cuda:0'), 'base_model.model.decoder.block.8.layer.1.EncDecAttention.v.lora_B.weight': tensor([[-0.0121,  0.0075,  0.0079,  ...,  0.0105,  0.0129, -0.0121],
        [ 0.0100,  0.0012, -0.0099,  ..., -0.0121, -0.0096,  0.0088],
        [ 0.0122, -0.0076, -0.0110,  ..., -0.0120, -0.0115,  0.0121],
        ...,
        [ 0.0093, -0.0080, -0.0108,  ..., -0.0075, -0.0032, -0.0013],
        [-0.0115,  0.0213,  0.0142,  ...,  0.0134,  0.0117, -0.0052],
        [-0.0036,  0.0162,  0.0016,  ...,  0.0018, -0.0023,  0.0021]],
       device='cuda:0'), 'base_model.model.decoder.block.9.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0154,  0.0037,  0.0349,  ..., -0.0158,  0.0038, -0.0385],
        [-0.0073,  0.0473, -0.0104,  ..., -0.0131, -0.0052, -0.0014],
        [ 0.0135, -0.0418,  0.0044,  ...,  0.0084,  0.0183, -0.0136],
        ...,
        [ 0.0269,  0.0013,  0.0371,  ..., -0.0096,  0.0070, -0.0114],
        [-0.0020, -0.0478,  0.0082,  ..., -0.0092,  0.0111, -0.0221],
        [ 0.0107, -0.0073, -0.0224,  ...,  0.0339, -0.0262,  0.0477]],
       device='cuda:0'), 'base_model.model.decoder.block.9.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0193,  0.0199, -0.0044,  ..., -0.0231, -0.0193,  0.0192],
        [-0.0032,  0.0015, -0.0103,  ...,  0.0019,  0.0071,  0.0039],
        [-0.0046,  0.0031, -0.0092,  ..., -0.0005,  0.0053,  0.0053],
        ...,
        [-0.0172,  0.0136,  0.0109,  ..., -0.0211, -0.0160,  0.0031],
        [-0.0070,  0.0049, -0.0063,  ..., -0.0058, -0.0043,  0.0051],
        [-0.0118,  0.0162, -0.0009,  ..., -0.0190, -0.0135,  0.0053]],
       device='cuda:0'), 'base_model.model.decoder.block.9.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0438, -0.0172,  0.0363,  ...,  0.0086, -0.0087,  0.0066],
        [ 0.0410, -0.0098,  0.0053,  ..., -0.0250,  0.0264,  0.0233],
        [ 0.0430, -0.0061,  0.0150,  ...,  0.0156,  0.0056, -0.0025],
        ...,
        [ 0.0078, -0.0022,  0.0056,  ..., -0.0328,  0.0030, -0.0042],
        [ 0.0094, -0.0351, -0.0269,  ..., -0.0067,  0.0339,  0.0215],
        [ 0.0293,  0.0221,  0.0090,  ...,  0.0187,  0.0156,  0.0453]],
       device='cuda:0'), 'base_model.model.decoder.block.9.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0210, -0.0218, -0.0219,  ...,  0.0225, -0.0235, -0.0230],
        [-0.0032,  0.0079, -0.0071,  ..., -0.0083,  0.0067,  0.0079],
        [ 0.0085, -0.0037, -0.0048,  ...,  0.0048, -0.0062, -0.0076],
        ...,
        [-0.0054,  0.0032,  0.0100,  ..., -0.0020,  0.0055,  0.0025],
        [-0.0110,  0.0081,  0.0158,  ..., -0.0079,  0.0112,  0.0075],
        [-0.0049,  0.0053,  0.0072,  ..., -0.0049,  0.0068,  0.0042]],
       device='cuda:0'), 'base_model.model.decoder.block.9.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0071, -0.0169, -0.0282,  ...,  0.0228,  0.0082, -0.0134],
        [-0.0093, -0.0039, -0.0467,  ...,  0.0432,  0.0402,  0.0078],
        [ 0.0147,  0.0395,  0.0144,  ..., -0.0338, -0.0109,  0.0084],
        ...,
        [ 0.0166, -0.0008,  0.0265,  ..., -0.0120, -0.0385, -0.0455],
        [ 0.0310, -0.0093,  0.0134,  ..., -0.0040, -0.0034, -0.0137],
        [-0.0219, -0.0405, -0.0077,  ...,  0.0289, -0.0082, -0.0013]],
       device='cuda:0'), 'base_model.model.decoder.block.9.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0050, -0.0137,  0.0062,  ...,  0.0020, -0.0027, -0.0102],
        [-0.0208,  0.0056, -0.0130,  ..., -0.0062,  0.0127,  0.0047],
        [-0.0034, -0.0049,  0.0036,  ...,  0.0025,  0.0036,  0.0013],
        ...,
        [ 0.0123, -0.0268,  0.0155,  ...,  0.0150, -0.0170, -0.0189],
        [-0.0296,  0.0265, -0.0202,  ..., -0.0150,  0.0275,  0.0163],
        [-0.0052,  0.0020, -0.0008,  ..., -0.0013,  0.0061,  0.0013]],
       device='cuda:0'), 'base_model.model.decoder.block.9.layer.1.EncDecAttention.v.lora_A.weight': tensor([[ 0.0483,  0.0039,  0.0312,  ..., -0.0194,  0.0217,  0.0157],
        [-0.0274,  0.0153,  0.0293,  ...,  0.0371, -0.0078, -0.0204],
        [-0.0226,  0.0241, -0.0100,  ...,  0.0308, -0.0354, -0.0683],
        ...,
        [ 0.0043,  0.0249, -0.0220,  ..., -0.0079,  0.0153,  0.0055],
        [ 0.0618,  0.0091,  0.0140,  ..., -0.0128,  0.0089,  0.0166],
        [-0.0502, -0.0238,  0.0167,  ...,  0.0662,  0.0163, -0.0201]],
       device='cuda:0'), 'base_model.model.decoder.block.9.layer.1.EncDecAttention.v.lora_B.weight': tensor([[-0.0136,  0.0160,  0.0156,  ..., -0.0165, -0.0171,  0.0161],
        [ 0.0025, -0.0026, -0.0052,  ...,  0.0055,  0.0045, -0.0020],
        [-0.0057,  0.0066,  0.0048,  ..., -0.0033, -0.0056,  0.0068],
        ...,
        [ 0.0240, -0.0242, -0.0254,  ...,  0.0231,  0.0241, -0.0242],
        [ 0.0213, -0.0170, -0.0214,  ...,  0.0198,  0.0180, -0.0202],
        [-0.0150,  0.0218,  0.0179,  ..., -0.0128, -0.0194,  0.0167]],
       device='cuda:0'), 'base_model.model.decoder.block.10.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0025, -0.0290,  0.0255,  ..., -0.0181,  0.0138,  0.0025],
        [-0.0440, -0.0035,  0.0296,  ...,  0.0243,  0.0088, -0.0053],
        [-0.0272,  0.0054, -0.0146,  ...,  0.0257,  0.0243,  0.0020],
        ...,
        [ 0.0010,  0.0344,  0.0134,  ...,  0.0117, -0.0007, -0.0075],
        [ 0.0423, -0.0215,  0.0008,  ..., -0.0328, -0.0238, -0.0170],
        [-0.0181,  0.0097,  0.0098,  ...,  0.0165, -0.0156, -0.0103]],
       device='cuda:0'), 'base_model.model.decoder.block.10.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0063,  0.0127,  0.0105,  ...,  0.0136,  0.0137,  0.0123],
        [ 0.0126, -0.0181, -0.0162,  ..., -0.0186, -0.0186, -0.0181],
        [-0.0070,  0.0156,  0.0164,  ...,  0.0148,  0.0144,  0.0143],
        ...,
        [-0.0039,  0.0014,  0.0003,  ...,  0.0025, -0.0060,  0.0027],
        [ 0.0025,  0.0164,  0.0130,  ...,  0.0138, -0.0010,  0.0125],
        [ 0.0102,  0.0055,  0.0062,  ..., -0.0012, -0.0060,  0.0018]],
       device='cuda:0'), 'base_model.model.decoder.block.10.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0024, -0.0390, -0.0036,  ...,  0.0109, -0.0263,  0.0339],
        [ 0.0104, -0.0131,  0.0153,  ..., -0.0151,  0.0249,  0.0227],
        [-0.0228,  0.0018,  0.0311,  ..., -0.0179,  0.0120, -0.0137],
        ...,
        [ 0.0398, -0.0251,  0.0051,  ...,  0.0207,  0.0257,  0.0066],
        [ 0.0026,  0.0058, -0.0247,  ...,  0.0021,  0.0233, -0.0101],
        [ 0.0265,  0.0026,  0.0404,  ...,  0.0282, -0.0161,  0.0450]],
       device='cuda:0'), 'base_model.model.decoder.block.10.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0041,  0.0035, -0.0041,  ...,  0.0040,  0.0048,  0.0029],
        [-0.0153, -0.0180,  0.0142,  ..., -0.0159, -0.0138, -0.0151],
        [ 0.0144,  0.0131, -0.0153,  ...,  0.0147,  0.0151,  0.0122],
        ...,
        [-0.0130, -0.0103,  0.0154,  ..., -0.0113, -0.0158, -0.0153],
        [-0.0170, -0.0153,  0.0144,  ..., -0.0125, -0.0155, -0.0185],
        [-0.0158, -0.0154,  0.0174,  ..., -0.0161, -0.0178, -0.0153]],
       device='cuda:0'), 'base_model.model.decoder.block.10.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0304, -0.0075, -0.0230,  ...,  0.0063, -0.0018,  0.0278],
        [-0.0076, -0.0079,  0.0015,  ..., -0.0316,  0.0023,  0.0243],
        [ 0.0084,  0.0124,  0.0012,  ...,  0.0109, -0.0317,  0.0220],
        ...,
        [-0.0283,  0.0414, -0.0127,  ..., -0.0159,  0.0182,  0.0143],
        [-0.0043, -0.0115, -0.0229,  ...,  0.0092, -0.0283,  0.0275],
        [-0.0176,  0.0361, -0.0088,  ..., -0.0113, -0.0109,  0.0135]],
       device='cuda:0'), 'base_model.model.decoder.block.10.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 8.9059e-03,  5.9997e-03,  3.2296e-03,  ...,  4.1912e-03,
         -7.3702e-04,  3.2630e-03],
        [ 1.4178e-02,  1.5074e-02,  8.4570e-03,  ...,  3.8711e-03,
          1.0596e-02,  6.3549e-03],
        [-1.2284e-02, -9.7453e-03,  5.3009e-04,  ...,  8.9120e-03,
          5.1250e-04, -5.0001e-03],
        ...,
        [ 4.1886e-03,  8.0591e-05, -2.7274e-04,  ...,  8.2174e-03,
         -2.2814e-03,  4.5755e-04],
        [ 1.7286e-02,  1.7134e-02,  1.1415e-02,  ...,  2.6513e-02,
          2.5976e-03,  1.1181e-02],
        [ 1.2990e-02,  1.4693e-02,  1.0798e-02,  ...,  1.6177e-02,
          5.8300e-03,  5.1070e-03]], device='cuda:0'), 'base_model.model.decoder.block.10.layer.1.EncDecAttention.v.lora_A.weight': tensor([[ 0.0020, -0.0097, -0.0073,  ..., -0.0110,  0.0074,  0.0097],
        [-0.0227,  0.0208, -0.0257,  ..., -0.0031, -0.0147, -0.0224],
        [-0.0020,  0.0450,  0.0228,  ..., -0.0019, -0.0001,  0.0342],
        ...,
        [-0.0067, -0.0084,  0.0036,  ...,  0.0493, -0.0207, -0.0334],
        [ 0.0320, -0.0389, -0.0380,  ...,  0.0327, -0.0444,  0.0107],
        [-0.0115, -0.0011,  0.0172,  ...,  0.0082, -0.0172, -0.0654]],
       device='cuda:0'), 'base_model.model.decoder.block.10.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 0.0213,  0.0044,  0.0228,  ..., -0.0223, -0.0208, -0.0105],
        [-0.0181,  0.0040, -0.0209,  ...,  0.0195,  0.0192,  0.0131],
        [ 0.0149,  0.0117,  0.0133,  ..., -0.0163, -0.0158, -0.0026],
        ...,
        [ 0.0104,  0.0159,  0.0134,  ..., -0.0119, -0.0154, -0.0064],
        [-0.0195, -0.0169, -0.0204,  ...,  0.0204,  0.0179,  0.0020],
        [-0.0017,  0.0049, -0.0006,  ..., -0.0002,  0.0053,  0.0112]],
       device='cuda:0'), 'base_model.model.decoder.block.11.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0113, -0.0173, -0.0027,  ...,  0.0034,  0.0462, -0.0084],
        [-0.0225, -0.0064, -0.0297,  ..., -0.0134,  0.0226,  0.0151],
        [ 0.0253, -0.0319,  0.0379,  ...,  0.0115,  0.0016, -0.0272],
        ...,
        [ 0.0177,  0.0429, -0.0232,  ...,  0.0427,  0.0081,  0.0265],
        [-0.0095, -0.0153, -0.0117,  ..., -0.0021, -0.0487, -0.0410],
        [ 0.0021,  0.0395,  0.0165,  ...,  0.0390,  0.0358,  0.0250]],
       device='cuda:0'), 'base_model.model.decoder.block.11.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0042, -0.0051,  0.0075,  ..., -0.0086,  0.0048, -0.0102],
        [-0.0015, -0.0072,  0.0053,  ..., -0.0051,  0.0027, -0.0075],
        [ 0.0172,  0.0165, -0.0048,  ...,  0.0060, -0.0144,  0.0042],
        ...,
        [-0.0212, -0.0192,  0.0205,  ..., -0.0172,  0.0160, -0.0143],
        [ 0.0145,  0.0102, -0.0123,  ...,  0.0128, -0.0037,  0.0073],
        [-0.0072, -0.0060,  0.0064,  ..., -0.0072,  0.0052, -0.0087]],
       device='cuda:0'), 'base_model.model.decoder.block.11.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0115, -0.0157,  0.0200,  ..., -0.0234,  0.0127,  0.0111],
        [ 0.0194,  0.0063, -0.0089,  ..., -0.0268, -0.0184, -0.0171],
        [ 0.0094, -0.0173,  0.0352,  ..., -0.0146, -0.0239,  0.0131],
        ...,
        [ 0.0015, -0.0065, -0.0278,  ..., -0.0262,  0.0022, -0.0093],
        [ 0.0198, -0.0332, -0.0144,  ...,  0.0026,  0.0404,  0.0179],
        [ 0.0105, -0.0364,  0.0262,  ..., -0.0171,  0.0403,  0.0121]],
       device='cuda:0'), 'base_model.model.decoder.block.11.layer.0.SelfAttention.v.lora_B.weight': tensor([[-9.5707e-03, -1.2350e-02,  4.2512e-03,  ..., -6.5697e-03,
          1.0621e-02,  8.1075e-03],
        [-1.3202e-03,  1.8115e-02, -1.6808e-02,  ...,  1.9649e-02,
         -1.0918e-02, -1.9214e-02],
        [ 1.8847e-02, -1.7127e-02,  1.7074e-02,  ..., -1.9088e-02,
          8.5360e-03,  1.7337e-02],
        ...,
        [-2.9112e-03, -2.2279e-02,  2.3047e-02,  ..., -2.6989e-02,
          2.1065e-02,  2.5149e-02],
        [ 1.2249e-02, -5.4296e-04,  1.3645e-02,  ..., -7.0064e-04,
         -1.2102e-03, -8.9333e-05],
        [-3.2261e-03,  1.1069e-02, -3.9940e-03,  ...,  1.3339e-02,
         -1.5026e-02, -1.2184e-02]], device='cuda:0'), 'base_model.model.decoder.block.11.layer.1.EncDecAttention.q.lora_A.weight': tensor([[ 0.0151,  0.0504,  0.0151,  ..., -0.0405, -0.0372,  0.0045],
        [ 0.0058, -0.0132, -0.0114,  ...,  0.0315, -0.0070, -0.0143],
        [ 0.0007, -0.0070, -0.0318,  ..., -0.0215,  0.0071,  0.0102],
        ...,
        [-0.0004,  0.0088,  0.0068,  ...,  0.0079, -0.0094,  0.0206],
        [-0.0109,  0.0159, -0.0031,  ..., -0.0096, -0.0329, -0.0108],
        [-0.0080,  0.0129,  0.0347,  ..., -0.0115,  0.0031,  0.0019]],
       device='cuda:0'), 'base_model.model.decoder.block.11.layer.1.EncDecAttention.q.lora_B.weight': tensor([[-0.0131,  0.0158,  0.0049,  ...,  0.0131, -0.0112, -0.0145],
        [ 0.0174, -0.0180, -0.0168,  ..., -0.0093,  0.0182,  0.0186],
        [-0.0105,  0.0116,  0.0139,  ..., -0.0073, -0.0136, -0.0119],
        ...,
        [-0.0058,  0.0093,  0.0242,  ..., -0.0101, -0.0032,  0.0060],
        [ 0.0191, -0.0159, -0.0204,  ...,  0.0079,  0.0200,  0.0092],
        [ 0.0009, -0.0016,  0.0067,  ...,  0.0027, -0.0018,  0.0032]],
       device='cuda:0'), 'base_model.model.decoder.block.11.layer.1.EncDecAttention.v.lora_A.weight': tensor([[ 0.0233,  0.0394, -0.0297,  ...,  0.0140, -0.0410, -0.0096],
        [-0.0128, -0.0103,  0.0050,  ..., -0.0203,  0.0273, -0.0021],
        [ 0.0241,  0.0009, -0.0340,  ..., -0.0166, -0.0095,  0.0592],
        ...,
        [ 0.0344,  0.0115, -0.0103,  ..., -0.0338,  0.0280,  0.0553],
        [-0.0379, -0.0005,  0.0447,  ..., -0.0204,  0.0497, -0.0573],
        [ 0.0310, -0.0138, -0.0048,  ...,  0.0067, -0.0408,  0.0392]],
       device='cuda:0'), 'base_model.model.decoder.block.11.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 2.2074e-02, -2.3050e-02, -2.0557e-02,  ..., -3.0574e-02,
          6.5095e-04,  7.4577e-03],
        [ 1.9601e-02, -1.9351e-02, -1.3357e-02,  ..., -1.9333e-02,
         -5.8716e-03,  1.7343e-02],
        [-2.0707e-02,  2.1192e-02,  1.7973e-02,  ...,  2.2283e-02,
          7.3185e-03, -1.4129e-02],
        ...,
        [-1.2650e-02,  5.2220e-03,  2.7920e-04,  ...,  1.1320e-03,
          2.8257e-03, -8.0793e-03],
        [-9.3410e-03, -1.6412e-03, -1.1426e-02,  ..., -1.1142e-02,
          5.2212e-03, -1.3261e-03],
        [-1.8118e-02,  1.8327e-02, -1.9824e-05,  ...,  2.1927e-02,
          8.2317e-03, -1.1294e-02]], device='cuda:0'), 'base_model.model.decoder.block.12.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0103,  0.0235, -0.0445,  ..., -0.0011,  0.0211,  0.0054],
        [ 0.0231,  0.0272, -0.0111,  ..., -0.0416, -0.0326, -0.0198],
        [-0.0178, -0.0060,  0.0432,  ..., -0.0134,  0.0074,  0.0074],
        ...,
        [ 0.0253,  0.0366,  0.0049,  ..., -0.0379, -0.0321,  0.0107],
        [-0.0126, -0.0017, -0.0359,  ...,  0.0121,  0.0077,  0.0259],
        [ 0.0278,  0.0117, -0.0050,  ...,  0.0169,  0.0003,  0.0109]],
       device='cuda:0'), 'base_model.model.decoder.block.12.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0189, -0.0169, -0.0191,  ..., -0.0217,  0.0187, -0.0174],
        [-0.0155,  0.0188,  0.0232,  ...,  0.0169, -0.0256,  0.0255],
        [-0.0193,  0.0153,  0.0159,  ...,  0.0276, -0.0155,  0.0142],
        ...,
        [-0.0200,  0.0177,  0.0180,  ...,  0.0193, -0.0177,  0.0189],
        [-0.0146,  0.0139,  0.0137,  ...,  0.0132, -0.0135,  0.0142],
        [ 0.0102, -0.0118, -0.0130,  ..., -0.0131,  0.0132, -0.0101]],
       device='cuda:0'), 'base_model.model.decoder.block.12.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 2.4031e-03,  2.1623e-02,  9.9847e-03,  ...,  1.5630e-02,
         -8.4626e-03, -1.9406e-02],
        [ 6.6478e-03,  2.4983e-02, -1.5062e-02,  ...,  1.9716e-02,
         -1.7178e-02,  1.3483e-03],
        [ 9.8836e-03,  6.9536e-03,  2.2963e-02,  ...,  3.3474e-02,
          2.1320e-02,  3.9549e-02],
        ...,
        [-2.3796e-02, -7.5400e-06,  3.0448e-02,  ...,  4.2281e-02,
          2.4435e-02,  3.4439e-03],
        [-8.6579e-03,  2.4908e-02, -1.4695e-02,  ..., -1.7937e-02,
         -2.9916e-02, -7.0838e-03],
        [-1.8854e-02, -4.6361e-02,  4.9669e-02,  ..., -1.4553e-04,
          2.3075e-02,  1.9088e-03]], device='cuda:0'), 'base_model.model.decoder.block.12.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0074, -0.0017,  0.0112,  ...,  0.0109, -0.0103,  0.0119],
        [-0.0149, -0.0008,  0.0157,  ...,  0.0172, -0.0093,  0.0181],
        [ 0.0007,  0.0051, -0.0025,  ..., -0.0014,  0.0031, -0.0015],
        ...,
        [-0.0013, -0.0033, -0.0026,  ..., -0.0015, -0.0034, -0.0038],
        [-0.0056, -0.0132,  0.0050,  ...,  0.0036, -0.0130, -0.0087],
        [-0.0098, -0.0120,  0.0107,  ...,  0.0110, -0.0117,  0.0023]],
       device='cuda:0'), 'base_model.model.decoder.block.12.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0046,  0.0211,  0.0200,  ..., -0.0087,  0.0239,  0.0120],
        [-0.0436, -0.0509, -0.0096,  ..., -0.0034,  0.0071, -0.0181],
        [-0.0019, -0.0168, -0.0081,  ..., -0.0142, -0.0002,  0.0396],
        ...,
        [ 0.0069, -0.0059,  0.0063,  ..., -0.0305,  0.0049,  0.0234],
        [-0.0008, -0.0510, -0.0299,  ...,  0.0231,  0.0161,  0.0187],
        [ 0.0062, -0.0050, -0.0067,  ...,  0.0040, -0.0048, -0.0111]],
       device='cuda:0'), 'base_model.model.decoder.block.12.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0257, -0.0312, -0.0249,  ..., -0.0225, -0.0249,  0.0227],
        [ 0.0275, -0.0350, -0.0293,  ..., -0.0257, -0.0283,  0.0259],
        [ 0.0032, -0.0025,  0.0020,  ..., -0.0031, -0.0025,  0.0046],
        ...,
        [-0.0013,  0.0057,  0.0068,  ...,  0.0017,  0.0016, -0.0019],
        [ 0.0046, -0.0059, -0.0001,  ..., -0.0073, -0.0064,  0.0085],
        [-0.0058,  0.0037,  0.0097,  ...,  0.0026,  0.0047, -0.0013]],
       device='cuda:0'), 'base_model.model.decoder.block.12.layer.1.EncDecAttention.v.lora_A.weight': tensor([[ 0.0103, -0.0327, -0.0249,  ...,  0.0200, -0.0102,  0.0046],
        [ 0.0150,  0.0122, -0.0172,  ...,  0.0032,  0.0125, -0.0109],
        [-0.0103,  0.0146,  0.0295,  ..., -0.0434, -0.0173, -0.0248],
        ...,
        [ 0.0349, -0.0145,  0.0197,  ...,  0.0345, -0.0413,  0.0070],
        [-0.0015, -0.0148,  0.0261,  ..., -0.0024, -0.0134, -0.0016],
        [ 0.0220,  0.0160,  0.0229,  ..., -0.0040, -0.0127,  0.0081]],
       device='cuda:0'), 'base_model.model.decoder.block.12.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 0.0108,  0.0126, -0.0129,  ...,  0.0061, -0.0068,  0.0116],
        [-0.0031, -0.0131,  0.0072,  ..., -0.0232,  0.0122, -0.0168],
        [-0.0215, -0.0097,  0.0125,  ..., -0.0147,  0.0112, -0.0142],
        ...,
        [-0.0178, -0.0196,  0.0217,  ..., -0.0212,  0.0185, -0.0224],
        [ 0.0107, -0.0059,  0.0002,  ..., -0.0074,  0.0075, -0.0123],
        [ 0.0042, -0.0160, -0.0046,  ..., -0.0154,  0.0115, -0.0159]],
       device='cuda:0'), 'base_model.model.decoder.block.13.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0219,  0.0094, -0.0227,  ..., -0.0118, -0.0069, -0.0019],
        [ 0.0170,  0.0005,  0.0317,  ..., -0.0442, -0.0148, -0.0178],
        [-0.0057, -0.0064,  0.0218,  ..., -0.0492, -0.0141, -0.0176],
        ...,
        [ 0.0341, -0.0037,  0.0090,  ...,  0.0453,  0.0237,  0.0424],
        [ 0.0107, -0.0036,  0.0088,  ..., -0.0421, -0.0325, -0.0168],
        [ 0.0242, -0.0061, -0.0386,  ..., -0.0047, -0.0117, -0.0059]],
       device='cuda:0'), 'base_model.model.decoder.block.13.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0086, -0.0127, -0.0096,  ...,  0.0039, -0.0092,  0.0090],
        [ 0.0119, -0.0163, -0.0191,  ...,  0.0047, -0.0144,  0.0121],
        [-0.0091,  0.0139,  0.0143,  ..., -0.0050,  0.0106, -0.0094],
        ...,
        [ 0.0141, -0.0123, -0.0156,  ...,  0.0110, -0.0122,  0.0114],
        [-0.0024,  0.0034,  0.0031,  ..., -0.0013,  0.0014, -0.0071],
        [ 0.0017, -0.0023, -0.0021,  ..., -0.0023,  0.0005,  0.0037]],
       device='cuda:0'), 'base_model.model.decoder.block.13.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0047,  0.0136, -0.0285,  ..., -0.0433,  0.0030,  0.0144],
        [ 0.0114,  0.0227, -0.0303,  ...,  0.0104,  0.0045,  0.0162],
        [ 0.0358,  0.0059,  0.0081,  ..., -0.0286, -0.0211, -0.0286],
        ...,
        [ 0.0013, -0.0096,  0.0008,  ...,  0.0270, -0.0111,  0.0180],
        [ 0.0205,  0.0279, -0.0179,  ..., -0.0381, -0.0168, -0.0347],
        [ 0.0088, -0.0382,  0.0431,  ...,  0.0359, -0.0130,  0.0356]],
       device='cuda:0'), 'base_model.model.decoder.block.13.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0092, -0.0092, -0.0036,  ...,  0.0120, -0.0077,  0.0126],
        [ 0.0039,  0.0076,  0.0078,  ...,  0.0046,  0.0068,  0.0035],
        [-0.0306, -0.0290, -0.0167,  ...,  0.0265, -0.0283,  0.0298],
        ...,
        [ 0.0154,  0.0149,  0.0118,  ..., -0.0089,  0.0155, -0.0134],
        [-0.0122, -0.0120, -0.0107,  ...,  0.0109, -0.0126,  0.0120],
        [-0.0047, -0.0033,  0.0010,  ...,  0.0054,  0.0011,  0.0038]],
       device='cuda:0'), 'base_model.model.decoder.block.13.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0253, -0.0134, -0.0078,  ..., -0.0472, -0.0238, -0.0436],
        [ 0.0033, -0.0051,  0.0008,  ...,  0.0166, -0.0053,  0.0373],
        [-0.0360, -0.0351, -0.0227,  ...,  0.0108,  0.0331, -0.0146],
        ...,
        [-0.0030,  0.0173, -0.0300,  ...,  0.0246,  0.0367,  0.0359],
        [ 0.0126,  0.0130,  0.0141,  ...,  0.0443, -0.0219, -0.0240],
        [ 0.0219,  0.0208, -0.0084,  ...,  0.0185,  0.0198,  0.0202]],
       device='cuda:0'), 'base_model.model.decoder.block.13.layer.1.EncDecAttention.q.lora_B.weight': tensor([[-0.0225,  0.0218,  0.0237,  ...,  0.0197,  0.0006,  0.0226],
        [-0.0211,  0.0232,  0.0246,  ...,  0.0203,  0.0224,  0.0244],
        [-0.0058, -0.0001,  0.0041,  ...,  0.0010,  0.0181,  0.0170],
        ...,
        [ 0.0031,  0.0004,  0.0067,  ..., -0.0013,  0.0070,  0.0063],
        [ 0.0117, -0.0107, -0.0180,  ..., -0.0149, -0.0202, -0.0204],
        [-0.0079,  0.0058,  0.0041,  ...,  0.0028,  0.0070,  0.0060]],
       device='cuda:0'), 'base_model.model.decoder.block.13.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0245, -0.0014,  0.0118,  ...,  0.0183,  0.0520,  0.0118],
        [-0.0252,  0.0178,  0.0142,  ...,  0.0115,  0.0291,  0.0260],
        [-0.0256,  0.0298,  0.0250,  ..., -0.0366,  0.0294,  0.0026],
        ...,
        [ 0.0112,  0.0158, -0.0059,  ...,  0.0497, -0.0213, -0.0500],
        [-0.0259,  0.0207, -0.0143,  ..., -0.0326,  0.0212,  0.0302],
        [-0.0123,  0.0010, -0.0071,  ...,  0.0158, -0.0003, -0.0120]],
       device='cuda:0'), 'base_model.model.decoder.block.13.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 0.0100,  0.0163,  0.0209,  ..., -0.0165,  0.0191,  0.0183],
        [-0.0046, -0.0019, -0.0039,  ...,  0.0066, -0.0077, -0.0095],
        [ 0.0071,  0.0146,  0.0189,  ..., -0.0203,  0.0199,  0.0203],
        ...,
        [ 0.0047,  0.0073,  0.0046,  ..., -0.0153,  0.0060,  0.0022],
        [-0.0199, -0.0175, -0.0212,  ...,  0.0074, -0.0213, -0.0187],
        [-0.0100, -0.0072, -0.0100,  ...,  0.0194, -0.0130, -0.0081]],
       device='cuda:0'), 'base_model.model.decoder.block.14.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0374, -0.0102, -0.0033,  ...,  0.0444, -0.0081, -0.0211],
        [ 0.0251, -0.0035, -0.0038,  ..., -0.0019, -0.0239,  0.0165],
        [ 0.0151,  0.0200, -0.0244,  ...,  0.0094, -0.0323, -0.0069],
        ...,
        [ 0.0082,  0.0296, -0.0086,  ...,  0.0067, -0.0011,  0.0200],
        [ 0.0051, -0.0162,  0.0073,  ..., -0.0121, -0.0001, -0.0147],
        [-0.0156,  0.0238,  0.0216,  ..., -0.0370, -0.0026, -0.0052]],
       device='cuda:0'), 'base_model.model.decoder.block.14.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0076, -0.0004, -0.0016,  ...,  0.0082,  0.0001,  0.0090],
        [-0.0125, -0.0170, -0.0166,  ..., -0.0114,  0.0149, -0.0104],
        [ 0.0138,  0.0154,  0.0162,  ...,  0.0136, -0.0150,  0.0128],
        ...,
        [ 0.0039,  0.0059,  0.0069,  ...,  0.0038, -0.0016,  0.0073],
        [ 0.0080, -0.0075, -0.0050,  ..., -0.0102,  0.0071, -0.0042],
        [ 0.0025,  0.0050,  0.0025,  ...,  0.0080, -0.0047,  0.0008]],
       device='cuda:0'), 'base_model.model.decoder.block.14.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0017,  0.0335,  0.0202,  ..., -0.0248,  0.0198, -0.0257],
        [ 0.0159, -0.0008,  0.0156,  ...,  0.0062,  0.0148,  0.0142],
        [ 0.0241,  0.0451, -0.0204,  ..., -0.0211, -0.0325, -0.0450],
        ...,
        [ 0.0282,  0.0004, -0.0164,  ..., -0.0083, -0.0320,  0.0056],
        [-0.0371, -0.0190, -0.0167,  ...,  0.0097,  0.0287,  0.0375],
        [ 0.0385,  0.0255, -0.0023,  ..., -0.0265, -0.0068, -0.0086]],
       device='cuda:0'), 'base_model.model.decoder.block.14.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0089, -0.0147, -0.0117,  ..., -0.0144,  0.0151, -0.0129],
        [ 0.0181,  0.0294,  0.0288,  ...,  0.0267, -0.0299,  0.0301],
        [-0.0206, -0.0198, -0.0207,  ..., -0.0200,  0.0170, -0.0197],
        ...,
        [-0.0094,  0.0028,  0.0006,  ...,  0.0009, -0.0027, -0.0012],
        [-0.0130, -0.0167, -0.0158,  ..., -0.0158,  0.0143, -0.0159],
        [-0.0003,  0.0117,  0.0091,  ...,  0.0089, -0.0094,  0.0117]],
       device='cuda:0'), 'base_model.model.decoder.block.14.layer.1.EncDecAttention.q.lora_A.weight': tensor([[ 0.0170, -0.0165, -0.0077,  ...,  0.0260,  0.0332, -0.0158],
        [ 0.0055, -0.0062,  0.0312,  ...,  0.0198,  0.0379, -0.0031],
        [-0.0055, -0.0003, -0.0136,  ...,  0.0274,  0.0339,  0.0083],
        ...,
        [-0.0270, -0.0042,  0.0200,  ..., -0.0558, -0.0361, -0.0222],
        [-0.0186,  0.0363, -0.0226,  ..., -0.0413, -0.0183,  0.0342],
        [ 0.0233, -0.0100, -0.0039,  ...,  0.0253,  0.0414, -0.0166]],
       device='cuda:0'), 'base_model.model.decoder.block.14.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0069,  0.0116,  0.0013,  ..., -0.0092, -0.0087,  0.0109],
        [-0.0145, -0.0106, -0.0112,  ...,  0.0108,  0.0059, -0.0067],
        [ 0.0009,  0.0047,  0.0026,  ..., -0.0067, -0.0045,  0.0030],
        ...,
        [-0.0020,  0.0029,  0.0040,  ..., -0.0054, -0.0019,  0.0033],
        [ 0.0104,  0.0127,  0.0050,  ..., -0.0121, -0.0120,  0.0125],
        [ 0.0132,  0.0152,  0.0163,  ..., -0.0161, -0.0142,  0.0154]],
       device='cuda:0'), 'base_model.model.decoder.block.14.layer.1.EncDecAttention.v.lora_A.weight': tensor([[ 0.0538, -0.0210, -0.0223,  ...,  0.0296,  0.0070,  0.0165],
        [ 0.0071,  0.0046, -0.0163,  ..., -0.0276, -0.0093,  0.0149],
        [ 0.0524,  0.0300, -0.0185,  ...,  0.0410, -0.0501, -0.0439],
        ...,
        [ 0.0027, -0.0458,  0.0228,  ..., -0.0456,  0.0372, -0.0044],
        [-0.0516, -0.0328,  0.0084,  ..., -0.0020,  0.0101, -0.0217],
        [-0.0488, -0.0184, -0.0100,  ..., -0.0236, -0.0005,  0.0046]],
       device='cuda:0'), 'base_model.model.decoder.block.14.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 0.0304, -0.0266,  0.0291,  ..., -0.0293, -0.0279, -0.0280],
        [ 0.0024, -0.0023,  0.0030,  ..., -0.0006,  0.0019, -0.0014],
        [-0.0144,  0.0198, -0.0207,  ...,  0.0224,  0.0214,  0.0192],
        ...,
        [ 0.0128, -0.0158,  0.0149,  ..., -0.0154, -0.0151, -0.0157],
        [ 0.0039, -0.0054,  0.0027,  ..., -0.0020, -0.0050, -0.0052],
        [ 0.0104, -0.0127,  0.0108,  ..., -0.0100, -0.0103, -0.0124]],
       device='cuda:0'), 'base_model.model.decoder.block.15.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0160, -0.0093,  0.0195,  ..., -0.0205, -0.0025, -0.0271],
        [ 0.0169, -0.0051,  0.0047,  ...,  0.0031,  0.0108,  0.0349],
        [-0.0346,  0.0079,  0.0012,  ..., -0.0336, -0.0087, -0.0240],
        ...,
        [ 0.0279, -0.0023, -0.0044,  ...,  0.0205, -0.0074,  0.0415],
        [ 0.0306, -0.0036, -0.0323,  ..., -0.0290,  0.0356,  0.0230],
        [-0.0206,  0.0021,  0.0137,  ..., -0.0037,  0.0064, -0.0108]],
       device='cuda:0'), 'base_model.model.decoder.block.15.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0021,  0.0034, -0.0017,  ...,  0.0019,  0.0005, -0.0005],
        [ 0.0027,  0.0014,  0.0029,  ..., -0.0032, -0.0004,  0.0007],
        [ 0.0128, -0.0122,  0.0117,  ..., -0.0105, -0.0078,  0.0099],
        ...,
        [ 0.0084, -0.0047,  0.0049,  ..., -0.0058, -0.0098,  0.0095],
        [ 0.0076, -0.0041,  0.0042,  ..., -0.0087, -0.0084,  0.0071],
        [-0.0067, -0.0047, -0.0006,  ...,  0.0078,  0.0089, -0.0042]],
       device='cuda:0'), 'base_model.model.decoder.block.15.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0054, -0.0268,  0.0230,  ...,  0.0257, -0.0094, -0.0198],
        [-0.0194,  0.0296,  0.0148,  ..., -0.0157, -0.0334, -0.0433],
        [-0.0143,  0.0536,  0.0044,  ..., -0.0396, -0.0147, -0.0384],
        ...,
        [-0.0108,  0.0081,  0.0216,  ..., -0.0186, -0.0146,  0.0007],
        [-0.0073, -0.0274, -0.0106,  ..., -0.0029, -0.0138,  0.0210],
        [-0.0089, -0.0516,  0.0305,  ...,  0.0103,  0.0208, -0.0060]],
       device='cuda:0'), 'base_model.model.decoder.block.15.layer.0.SelfAttention.v.lora_B.weight': tensor([[-7.7887e-03, -4.4329e-03, -1.0258e-02,  ..., -1.1257e-02,
          9.5218e-03,  1.0012e-02],
        [-1.3725e-02, -1.3488e-02, -2.1645e-03,  ..., -7.9241e-04,
          2.8600e-03,  2.6073e-03],
        [-1.9941e-02, -5.7291e-03, -3.8434e-03,  ..., -4.3191e-03,
          7.2316e-03,  3.0308e-03],
        ...,
        [-2.7335e-03, -2.2611e-02, -1.1037e-02,  ..., -9.8621e-03,
          1.4747e-02,  1.1001e-02],
        [-1.6099e-02, -1.4242e-03, -2.9456e-03,  ...,  6.2675e-04,
          5.8403e-05,  3.8604e-03],
        [-1.2524e-02, -1.1022e-02, -1.3438e-02,  ..., -1.4034e-02,
          1.6053e-02,  1.5081e-02]], device='cuda:0'), 'base_model.model.decoder.block.15.layer.1.EncDecAttention.q.lora_A.weight': tensor([[ 0.0230,  0.0099,  0.0328,  ..., -0.0349,  0.0029,  0.0024],
        [ 0.0306,  0.0177, -0.0057,  ..., -0.0538, -0.0250,  0.0284],
        [-0.0387, -0.0495,  0.0077,  ...,  0.0302, -0.0180,  0.0018],
        ...,
        [-0.0281, -0.0137, -0.0024,  ...,  0.0016,  0.0356,  0.0271],
        [-0.0477, -0.0220,  0.0125,  ...,  0.0446,  0.0032, -0.0169],
        [-0.0168, -0.0307, -0.0202,  ...,  0.0207,  0.0262,  0.0194]],
       device='cuda:0'), 'base_model.model.decoder.block.15.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0084,  0.0045, -0.0095,  ..., -0.0095, -0.0065, -0.0033],
        [ 0.0140,  0.0250, -0.0206,  ..., -0.0109, -0.0197, -0.0172],
        [-0.0027, -0.0009,  0.0136,  ...,  0.0028, -0.0014,  0.0007],
        ...,
        [ 0.0039,  0.0058, -0.0080,  ..., -0.0043, -0.0039, -0.0040],
        [ 0.0005, -0.0047,  0.0135,  ...,  0.0073,  0.0052,  0.0081],
        [-0.0088, -0.0020,  0.0143,  ...,  0.0133,  0.0091,  0.0072]],
       device='cuda:0'), 'base_model.model.decoder.block.15.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0296, -0.0106,  0.0387,  ..., -0.0194, -0.0090, -0.0105],
        [ 0.0402,  0.0177,  0.0116,  ...,  0.0274,  0.0108, -0.0268],
        [-0.0306,  0.0236,  0.0116,  ..., -0.0145,  0.0411,  0.0181],
        ...,
        [-0.0032, -0.0044, -0.0173,  ...,  0.0181, -0.0130, -0.0105],
        [ 0.0389,  0.0156, -0.0165,  ...,  0.0283, -0.0330,  0.0074],
        [-0.0487,  0.0026,  0.0130,  ..., -0.0018,  0.0217, -0.0254]],
       device='cuda:0'), 'base_model.model.decoder.block.15.layer.1.EncDecAttention.v.lora_B.weight': tensor([[-0.0079,  0.0204, -0.0110,  ...,  0.0214,  0.0234, -0.0076],
        [ 0.0076, -0.0051,  0.0005,  ...,  0.0017, -0.0050,  0.0087],
        [-0.0129,  0.0082, -0.0114,  ...,  0.0144,  0.0113, -0.0152],
        ...,
        [-0.0073,  0.0115, -0.0072,  ...,  0.0007,  0.0094, -0.0099],
        [ 0.0005,  0.0024, -0.0036,  ...,  0.0100,  0.0121, -0.0115],
        [-0.0065, -0.0010, -0.0054,  ...,  0.0009,  0.0019, -0.0048]],
       device='cuda:0'), 'base_model.model.decoder.block.16.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0219, -0.0087, -0.0181,  ...,  0.0393,  0.0068, -0.0218],
        [-0.0422, -0.0278,  0.0179,  ...,  0.0154,  0.0026,  0.0087],
        [ 0.0098, -0.0264,  0.0298,  ...,  0.0034, -0.0046, -0.0290],
        ...,
        [ 0.0221, -0.0257,  0.0337,  ...,  0.0161, -0.0059, -0.0238],
        [-0.0034, -0.0086, -0.0082,  ..., -0.0122, -0.0405,  0.0123],
        [ 0.0056, -0.0267, -0.0337,  ...,  0.0398, -0.0013,  0.0338]],
       device='cuda:0'), 'base_model.model.decoder.block.16.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0045,  0.0062,  0.0051,  ..., -0.0066, -0.0064,  0.0069],
        [-0.0070, -0.0047, -0.0063,  ...,  0.0037,  0.0037, -0.0014],
        [ 0.0058,  0.0074, -0.0024,  ..., -0.0094, -0.0086,  0.0102],
        ...,
        [-0.0225, -0.0194,  0.0190,  ...,  0.0209,  0.0221, -0.0232],
        [-0.0137, -0.0164,  0.0105,  ...,  0.0161,  0.0164, -0.0072],
        [ 0.0036,  0.0069,  0.0028,  ..., -0.0018, -0.0002, -0.0341]],
       device='cuda:0'), 'base_model.model.decoder.block.16.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0219, -0.0161,  0.0182,  ...,  0.0006,  0.0182,  0.0124],
        [ 0.0040,  0.0343,  0.0194,  ..., -0.0299, -0.0435,  0.0343],
        [ 0.0039, -0.0184, -0.0296,  ..., -0.0075,  0.0109, -0.0545],
        ...,
        [ 0.0032,  0.0418, -0.0225,  ..., -0.0043, -0.0383, -0.0002],
        [ 0.0360,  0.0251,  0.0007,  ..., -0.0507, -0.0307,  0.0645],
        [ 0.0341,  0.0468,  0.0273,  ..., -0.0019, -0.0207,  0.0422]],
       device='cuda:0'), 'base_model.model.decoder.block.16.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0233, -0.0240,  0.0242,  ..., -0.0231, -0.0250, -0.0250],
        [-0.0088, -0.0091,  0.0081,  ..., -0.0089, -0.0099, -0.0091],
        [ 0.0041,  0.0020,  0.0018,  ...,  0.0014,  0.0007,  0.0016],
        ...,
        [-0.0078, -0.0184,  0.0138,  ..., -0.0104, -0.0209, -0.0149],
        [ 0.0161,  0.0129, -0.0111,  ...,  0.0140,  0.0109,  0.0143],
        [ 0.0067,  0.0030, -0.0056,  ...,  0.0036, -0.0010,  0.0049]],
       device='cuda:0'), 'base_model.model.decoder.block.16.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0027, -0.0303,  0.0067,  ..., -0.0229,  0.0209, -0.0409],
        [-0.0099, -0.0124,  0.0144,  ..., -0.0010, -0.0295,  0.0039],
        [-0.0113,  0.0164,  0.0156,  ...,  0.0317,  0.0263,  0.0163],
        ...,
        [ 0.0317, -0.0133,  0.0245,  ..., -0.0112,  0.0027,  0.0285],
        [ 0.0233,  0.0047, -0.0222,  ..., -0.0179,  0.0321,  0.0024],
        [ 0.0352, -0.0024, -0.0367,  ...,  0.0203, -0.0148, -0.0224]],
       device='cuda:0'), 'base_model.model.decoder.block.16.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0052, -0.0046,  0.0103,  ...,  0.0058,  0.0120,  0.0090],
        [-0.0022,  0.0010, -0.0026,  ..., -0.0087, -0.0056, -0.0035],
        [ 0.0039, -0.0125,  0.0089,  ..., -0.0214,  0.0074,  0.0066],
        ...,
        [ 0.0275, -0.0290,  0.0311,  ..., -0.0106,  0.0308,  0.0293],
        [ 0.0141, -0.0132,  0.0107,  ..., -0.0075,  0.0121,  0.0140],
        [-0.0304,  0.0323, -0.0312,  ...,  0.0101, -0.0350, -0.0371]],
       device='cuda:0'), 'base_model.model.decoder.block.16.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0211, -0.0161,  0.0199,  ..., -0.0291,  0.0098,  0.0633],
        [ 0.0079, -0.0033, -0.0019,  ...,  0.0192, -0.0015, -0.0375],
        [-0.0076, -0.0288,  0.0126,  ...,  0.0054, -0.0087,  0.0139],
        ...,
        [ 0.0009, -0.0255,  0.0367,  ..., -0.0258,  0.0083,  0.0221],
        [ 0.0277,  0.0011,  0.0027,  ..., -0.0070,  0.0137,  0.0379],
        [ 0.0215,  0.0060, -0.0134,  ...,  0.0053, -0.0184, -0.0192]],
       device='cuda:0'), 'base_model.model.decoder.block.16.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 0.0108,  0.0078,  0.0119,  ...,  0.0124,  0.0036, -0.0123],
        [ 0.0202, -0.0023,  0.0220,  ...,  0.0209, -0.0016, -0.0214],
        [-0.0082,  0.0027,  0.0075,  ..., -0.0011, -0.0155,  0.0050],
        ...,
        [-0.0056,  0.0035, -0.0025,  ..., -0.0039, -0.0004,  0.0047],
        [-0.0191,  0.0135, -0.0108,  ..., -0.0092, -0.0125,  0.0128],
        [ 0.0191, -0.0031,  0.0218,  ...,  0.0199,  0.0088, -0.0202]],
       device='cuda:0'), 'base_model.model.decoder.block.17.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0110,  0.0033, -0.0113,  ...,  0.0048,  0.0247,  0.0054],
        [ 0.0127, -0.0212, -0.0472,  ..., -0.0268,  0.0428, -0.0202],
        [-0.0078,  0.0169,  0.0111,  ..., -0.0042, -0.0003, -0.0235],
        ...,
        [-0.0244, -0.0122, -0.0304,  ..., -0.0329,  0.0367, -0.0051],
        [ 0.0205,  0.0283,  0.0346,  ...,  0.0034,  0.0209,  0.0203],
        [-0.0127,  0.0077,  0.0263,  ...,  0.0332, -0.0028,  0.0078]],
       device='cuda:0'), 'base_model.model.decoder.block.17.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0027,  0.0025,  0.0065,  ...,  0.0052, -0.0079, -0.0049],
        [ 0.0232,  0.0130,  0.0136,  ...,  0.0119, -0.0129, -0.0109],
        [-0.0266, -0.0229, -0.0219,  ..., -0.0216,  0.0204,  0.0209],
        ...,
        [ 0.0131,  0.0106,  0.0096,  ...,  0.0094, -0.0127, -0.0143],
        [ 0.0107,  0.0033,  0.0019,  ...,  0.0027, -0.0010, -0.0041],
        [-0.0081, -0.0046, -0.0037,  ..., -0.0038,  0.0019,  0.0018]],
       device='cuda:0'), 'base_model.model.decoder.block.17.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0325,  0.0404, -0.0134,  ...,  0.0062, -0.0371,  0.0013],
        [-0.0041, -0.0016,  0.0144,  ..., -0.0214, -0.0108, -0.0138],
        [-0.0005,  0.0322, -0.0368,  ..., -0.0403,  0.0035, -0.0059],
        ...,
        [ 0.0045,  0.0389,  0.0109,  ..., -0.0075, -0.0146,  0.0066],
        [-0.0320, -0.0356,  0.0048,  ...,  0.0524,  0.0445, -0.0348],
        [-0.0180, -0.0509,  0.0261,  ...,  0.0295,  0.0356, -0.0276]],
       device='cuda:0'), 'base_model.model.decoder.block.17.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0078,  0.0004,  0.0029,  ..., -0.0020,  0.0022,  0.0038],
        [ 0.0122,  0.0044,  0.0141,  ...,  0.0164, -0.0155, -0.0095],
        [-0.0067,  0.0061,  0.0006,  ..., -0.0035,  0.0040,  0.0004],
        ...,
        [ 0.0147,  0.0099,  0.0087,  ...,  0.0133, -0.0135, -0.0156],
        [ 0.0059, -0.0010,  0.0012,  ...,  0.0056, -0.0035,  0.0037],
        [ 0.0002,  0.0016,  0.0042,  ...,  0.0002,  0.0006,  0.0034]],
       device='cuda:0'), 'base_model.model.decoder.block.17.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0348, -0.0118,  0.0138,  ..., -0.0390,  0.0242, -0.0179],
        [ 0.0164, -0.0063,  0.0224,  ..., -0.0279,  0.0316, -0.0096],
        [ 0.0018, -0.0266,  0.0202,  ..., -0.0475, -0.0246, -0.0220],
        ...,
        [ 0.0014, -0.0097, -0.0064,  ...,  0.0311,  0.0035, -0.0014],
        [ 0.0395,  0.0291,  0.0073,  ..., -0.0023, -0.0247,  0.0419],
        [ 0.0226,  0.0164, -0.0280,  ...,  0.0056,  0.0016,  0.0153]],
       device='cuda:0'), 'base_model.model.decoder.block.17.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0078,  0.0093,  0.0078,  ..., -0.0093, -0.0051, -0.0021],
        [-0.0284, -0.0147, -0.0206,  ...,  0.0180,  0.0384,  0.0393],
        [ 0.0054,  0.0074,  0.0035,  ..., -0.0096, -0.0074, -0.0064],
        ...,
        [-0.0116, -0.0127, -0.0108,  ...,  0.0129,  0.0128,  0.0102],
        [ 0.0139,  0.0120,  0.0094,  ..., -0.0132, -0.0130, -0.0141],
        [ 0.0109,  0.0123,  0.0065,  ..., -0.0120, -0.0124, -0.0094]],
       device='cuda:0'), 'base_model.model.decoder.block.17.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0262, -0.0628,  0.0287,  ..., -0.0263,  0.0311, -0.0192],
        [-0.0029,  0.0124, -0.0072,  ...,  0.0043, -0.0331, -0.0429],
        [ 0.0236,  0.0187, -0.0174,  ...,  0.0047,  0.0140,  0.0135],
        ...,
        [ 0.0152,  0.0109,  0.0131,  ..., -0.0012, -0.0165,  0.0063],
        [-0.0361, -0.0315,  0.0343,  ..., -0.0198,  0.0440, -0.0010],
        [ 0.0114,  0.0118, -0.0288,  ...,  0.0051, -0.0095, -0.0169]],
       device='cuda:0'), 'base_model.model.decoder.block.17.layer.1.EncDecAttention.v.lora_B.weight': tensor([[-0.0318,  0.0044,  0.0069,  ...,  0.0107, -0.0189,  0.0074],
        [-0.0332,  0.0219, -0.0159,  ..., -0.0156, -0.0224,  0.0240],
        [ 0.0288, -0.0129,  0.0086,  ...,  0.0045,  0.0240, -0.0148],
        ...,
        [ 0.0157,  0.0096, -0.0117,  ..., -0.0127, -0.0148, -0.0194],
        [ 0.0155,  0.0098, -0.0099,  ..., -0.0118, -0.0114, -0.0147],
        [-0.0018, -0.0110,  0.0131,  ...,  0.0123,  0.0138,  0.0039]],
       device='cuda:0'), 'base_model.model.decoder.block.18.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0352, -0.0196,  0.0188,  ...,  0.0375, -0.0202, -0.0196],
        [ 0.0414,  0.0130, -0.0185,  ...,  0.0279, -0.0007, -0.0084],
        [ 0.0244, -0.0149, -0.0413,  ...,  0.0338, -0.0023,  0.0292],
        ...,
        [ 0.0105,  0.0305,  0.0255,  ..., -0.0105, -0.0306, -0.0135],
        [ 0.0201,  0.0205,  0.0201,  ...,  0.0055, -0.0065, -0.0440],
        [ 0.0362, -0.0236,  0.0030,  ..., -0.0126,  0.0508,  0.0348]],
       device='cuda:0'), 'base_model.model.decoder.block.18.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0081,  0.0077,  0.0071,  ..., -0.0068,  0.0024,  0.0021],
        [-0.0012,  0.0028,  0.0023,  ..., -0.0028, -0.0080,  0.0022],
        [ 0.0074,  0.0088,  0.0088,  ..., -0.0086,  0.0027,  0.0098],
        ...,
        [-0.0137, -0.0088, -0.0094,  ...,  0.0106,  0.0117, -0.0056],
        [ 0.0091,  0.0049,  0.0071,  ..., -0.0084,  0.0008, -0.0027],
        [-0.0011, -0.0091, -0.0071,  ...,  0.0047, -0.0046, -0.0082]],
       device='cuda:0'), 'base_model.model.decoder.block.18.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0202,  0.0472, -0.0144,  ..., -0.0219,  0.0103,  0.0043],
        [ 0.0089,  0.0365,  0.0118,  ..., -0.0073,  0.0052,  0.0454],
        [-0.0188,  0.0213, -0.0220,  ..., -0.0063, -0.0301,  0.0115],
        ...,
        [ 0.0001,  0.0202,  0.0150,  ..., -0.0270,  0.0132,  0.0644],
        [-0.0128,  0.0170,  0.0343,  ...,  0.0307,  0.0125,  0.0154],
        [ 0.0368, -0.0510, -0.0221,  ...,  0.0050,  0.0360, -0.0401]],
       device='cuda:0'), 'base_model.model.decoder.block.18.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0183,  0.0209,  0.0223,  ...,  0.0224, -0.0276, -0.0238],
        [ 0.0018,  0.0003, -0.0033,  ..., -0.0021,  0.0007,  0.0008],
        [-0.0094, -0.0069, -0.0029,  ..., -0.0034,  0.0021,  0.0041],
        ...,
        [ 0.0106,  0.0082, -0.0028,  ..., -0.0003,  0.0051,  0.0002],
        [ 0.0205,  0.0214,  0.0110,  ...,  0.0128, -0.0095, -0.0170],
        [-0.0337, -0.0304, -0.0236,  ..., -0.0311,  0.0311,  0.0223]],
       device='cuda:0'), 'base_model.model.decoder.block.18.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0321, -0.0146,  0.0019,  ...,  0.0236,  0.0183, -0.0471],
        [-0.0162, -0.0074, -0.0296,  ...,  0.0163, -0.0134, -0.0315],
        [ 0.0442, -0.0036, -0.0010,  ..., -0.0262,  0.0005,  0.0129],
        ...,
        [ 0.0039,  0.0022, -0.0167,  ..., -0.0317,  0.0070,  0.0184],
        [-0.0482, -0.0197, -0.0316,  ..., -0.0014, -0.0049, -0.0271],
        [ 0.0432, -0.0289,  0.0295,  ..., -0.0390, -0.0308,  0.0124]],
       device='cuda:0'), 'base_model.model.decoder.block.18.layer.1.EncDecAttention.q.lora_B.weight': tensor([[-0.0075, -0.0108,  0.0096,  ...,  0.0016, -0.0095,  0.0080],
        [-0.0140, -0.0120,  0.0073,  ...,  0.0127, -0.0119,  0.0123],
        [-0.0056, -0.0077,  0.0043,  ..., -0.0065, -0.0042,  0.0043],
        ...,
        [-0.0019,  0.0033,  0.0096,  ..., -0.0167, -0.0077,  0.0028],
        [-0.0132, -0.0107,  0.0170,  ...,  0.0081, -0.0119,  0.0058],
        [-0.0042, -0.0003,  0.0090,  ...,  0.0026, -0.0019,  0.0004]],
       device='cuda:0'), 'base_model.model.decoder.block.18.layer.1.EncDecAttention.v.lora_A.weight': tensor([[ 0.0196,  0.0030,  0.0224,  ..., -0.0038,  0.0045,  0.0215],
        [ 0.0100,  0.0074,  0.0359,  ...,  0.0210,  0.0242,  0.0113],
        [ 0.0026, -0.0086,  0.0127,  ...,  0.0132, -0.0101,  0.0025],
        ...,
        [ 0.0285, -0.0291,  0.0067,  ..., -0.0070, -0.0483, -0.0456],
        [-0.0019, -0.0021, -0.0288,  ..., -0.0177,  0.0375,  0.0036],
        [-0.0219,  0.0325,  0.0195,  ..., -0.0087, -0.0078,  0.0244]],
       device='cuda:0'), 'base_model.model.decoder.block.18.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 0.0184, -0.0062, -0.0195,  ..., -0.0182,  0.0185,  0.0185],
        [ 0.0150, -0.0073, -0.0114,  ..., -0.0022,  0.0037,  0.0024],
        [ 0.0261, -0.0178, -0.0264,  ..., -0.0250,  0.0274,  0.0250],
        ...,
        [ 0.0242, -0.0040, -0.0231,  ..., -0.0215,  0.0186,  0.0224],
        [-0.0210,  0.0187,  0.0201,  ...,  0.0210, -0.0218, -0.0205],
        [-0.0242, -0.0048,  0.0228,  ...,  0.0187, -0.0221, -0.0207]],
       device='cuda:0'), 'base_model.model.decoder.block.19.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0391, -0.0192,  0.0063,  ...,  0.0308,  0.0023,  0.0011],
        [-0.0155,  0.0229, -0.0224,  ..., -0.0024,  0.0059, -0.0193],
        [ 0.0280, -0.0313, -0.0107,  ..., -0.0039,  0.0153,  0.0012],
        ...,
        [ 0.0262,  0.0110, -0.0075,  ..., -0.0037,  0.0194, -0.0427],
        [-0.0242, -0.0382,  0.0120,  ...,  0.0300, -0.0108,  0.0073],
        [-0.0040, -0.0096,  0.0203,  ..., -0.0097, -0.0198, -0.0229]],
       device='cuda:0'), 'base_model.model.decoder.block.19.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 6.1453e-03,  6.1973e-03, -6.8584e-03,  ..., -4.5252e-03,
          6.3516e-03,  6.2948e-03],
        [-4.5412e-03, -4.4871e-03,  4.0846e-03,  ...,  5.9160e-03,
         -4.4922e-03, -4.3107e-03],
        [-6.2454e-03, -6.1674e-03,  6.7055e-03,  ...,  4.5486e-03,
         -6.3253e-03, -6.0943e-03],
        ...,
        [-6.4837e-03, -7.3207e-03,  8.4997e-03,  ..., -6.3963e-05,
         -3.4027e-04,  1.2465e-02],
        [ 1.0956e-02,  9.6466e-03, -7.7077e-03,  ..., -9.7397e-03,
          1.1114e-02, -2.4019e-02],
        [ 2.1932e-02,  1.3618e-02, -1.6683e-02,  ..., -1.7114e-02,
          1.7331e-02, -2.1152e-02]], device='cuda:0'), 'base_model.model.decoder.block.19.layer.0.SelfAttention.v.lora_A.weight': tensor([[-1.2191e-03, -1.8192e-02,  2.0656e-02,  ..., -3.0691e-02,
         -3.7022e-02,  6.2503e-02],
        [-8.9603e-05, -2.4246e-02, -1.6662e-02,  ..., -4.4149e-02,
         -7.4380e-03,  1.4131e-02],
        [-1.2862e-03,  2.0751e-02,  1.4463e-03,  ...,  1.3739e-02,
          3.9745e-02, -2.3024e-02],
        ...,
        [ 1.6170e-03,  2.8526e-02, -1.6437e-02,  ..., -4.2300e-02,
         -1.4941e-02,  4.1151e-02],
        [-1.9709e-02, -1.1424e-02,  1.7977e-02,  ...,  3.2824e-02,
          6.1655e-03, -5.5113e-02],
        [-4.6231e-03, -1.9727e-03,  2.2364e-02,  ..., -3.3645e-03,
         -1.6693e-02, -9.5787e-04]], device='cuda:0'), 'base_model.model.decoder.block.19.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0227,  0.0295, -0.0247,  ...,  0.0254, -0.0258,  0.0219],
        [ 0.0271,  0.0284, -0.0212,  ...,  0.0271, -0.0269,  0.0150],
        [ 0.0234,  0.0235, -0.0208,  ...,  0.0246, -0.0221,  0.0152],
        ...,
        [ 0.0078,  0.0068, -0.0066,  ...,  0.0072, -0.0073,  0.0119],
        [ 0.0140,  0.0122, -0.0158,  ...,  0.0121, -0.0126,  0.0089],
        [-0.0119, -0.0147,  0.0114,  ..., -0.0144,  0.0136, -0.0122]],
       device='cuda:0'), 'base_model.model.decoder.block.19.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0183,  0.0285, -0.0087,  ...,  0.0389, -0.0304, -0.0211],
        [-0.0469,  0.0042, -0.0137,  ...,  0.0184, -0.0258,  0.0023],
        [ 0.0009, -0.0048, -0.0200,  ...,  0.0073, -0.0202,  0.0201],
        ...,
        [-0.0237,  0.0142,  0.0049,  ...,  0.0029, -0.0308,  0.0134],
        [-0.0482,  0.0319, -0.0207,  ...,  0.0067, -0.0235, -0.0195],
        [ 0.0306, -0.0269,  0.0360,  ..., -0.0022, -0.0164,  0.0399]],
       device='cuda:0'), 'base_model.model.decoder.block.19.layer.1.EncDecAttention.q.lora_B.weight': tensor([[-0.0137, -0.0193,  0.0165,  ..., -0.0196, -0.0190,  0.0197],
        [ 0.0025, -0.0017,  0.0025,  ..., -0.0013, -0.0026,  0.0021],
        [-0.0140, -0.0209,  0.0177,  ..., -0.0156, -0.0172,  0.0192],
        ...,
        [ 0.0085,  0.0040, -0.0062,  ...,  0.0022,  0.0034, -0.0028],
        [-0.0045, -0.0048,  0.0089,  ..., -0.0104, -0.0076,  0.0103],
        [-0.0155, -0.0158,  0.0103,  ..., -0.0147, -0.0128,  0.0156]],
       device='cuda:0'), 'base_model.model.decoder.block.19.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0591, -0.0330,  0.0479,  ..., -0.0135,  0.0269, -0.0320],
        [ 0.0116,  0.0332,  0.0454,  ..., -0.0533,  0.0502,  0.0280],
        [ 0.0680,  0.0278, -0.0021,  ...,  0.0058, -0.0372,  0.0176],
        ...,
        [-0.0592, -0.0317,  0.0202,  ...,  0.0044,  0.0342, -0.0396],
        [-0.0679, -0.0569,  0.0012,  ...,  0.0163,  0.0423, -0.0169],
        [ 0.0339,  0.0452, -0.0357,  ...,  0.0423, -0.0517,  0.0218]],
       device='cuda:0'), 'base_model.model.decoder.block.19.layer.1.EncDecAttention.v.lora_B.weight': tensor([[-0.0204, -0.0194, -0.0167,  ..., -0.0192,  0.0034,  0.0193],
        [-0.0046, -0.0046, -0.0007,  ..., -0.0025,  0.0107,  0.0048],
        [-0.0109, -0.0102, -0.0077,  ..., -0.0106,  0.0037,  0.0111],
        ...,
        [ 0.0105, -0.0051, -0.0013,  ..., -0.0022, -0.0002,  0.0012],
        [ 0.0298,  0.0384, -0.0381,  ...,  0.0380,  0.0370, -0.0360],
        [-0.0161, -0.0247,  0.0242,  ..., -0.0196, -0.0242,  0.0225]],
       device='cuda:0'), 'base_model.model.decoder.block.20.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0330, -0.0107,  0.0211,  ...,  0.0107,  0.0013, -0.0105],
        [ 0.0142, -0.0134,  0.0259,  ...,  0.0185,  0.0278,  0.0195],
        [-0.0087, -0.0145, -0.0396,  ...,  0.0157, -0.0085, -0.0097],
        ...,
        [-0.0029,  0.0066, -0.0174,  ...,  0.0006, -0.0260, -0.0203],
        [ 0.0232,  0.0241,  0.0129,  ..., -0.0420,  0.0071, -0.0258],
        [ 0.0318,  0.0162,  0.0062,  ...,  0.0249,  0.0159, -0.0238]],
       device='cuda:0'), 'base_model.model.decoder.block.20.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0066,  0.0075,  0.0076,  ..., -0.0074, -0.0073, -0.0012],
        [-0.0060, -0.0079, -0.0063,  ...,  0.0062,  0.0067,  0.0021],
        [-0.0026, -0.0072, -0.0017,  ...,  0.0012,  0.0013, -0.0047],
        ...,
        [-0.0108, -0.0189, -0.0058,  ...,  0.0137,  0.0104,  0.0008],
        [-0.0061, -0.0086, -0.0066,  ...,  0.0030,  0.0012, -0.0078],
        [ 0.0148,  0.0042,  0.0149,  ..., -0.0157, -0.0143, -0.0109]],
       device='cuda:0'), 'base_model.model.decoder.block.20.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0134,  0.0197,  0.0471,  ..., -0.0059, -0.0069,  0.0288],
        [-0.0282, -0.0023, -0.0330,  ...,  0.0061,  0.0268, -0.0604],
        [-0.0080,  0.0056,  0.0321,  ..., -0.0319,  0.0065,  0.0143],
        ...,
        [-0.0269,  0.0138,  0.0644,  ...,  0.0223, -0.0018,  0.0240],
        [-0.0010, -0.0047,  0.0269,  ...,  0.0136,  0.0345, -0.0001],
        [-0.0323,  0.0287,  0.0055,  ...,  0.0195,  0.0189, -0.0179]],
       device='cuda:0'), 'base_model.model.decoder.block.20.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0061,  0.0025, -0.0070,  ...,  0.0109,  0.0056,  0.0067],
        [-0.0246,  0.0245, -0.0271,  ..., -0.0096,  0.0270,  0.0278],
        [ 0.0282, -0.0249,  0.0236,  ...,  0.0410, -0.0145, -0.0263],
        ...,
        [-0.0128,  0.0086,  0.0037,  ..., -0.0332, -0.0036,  0.0066],
        [-0.0181,  0.0180, -0.0174,  ..., -0.0017,  0.0149,  0.0184],
        [-0.0168,  0.0144, -0.0048,  ...,  0.0045,  0.0113,  0.0176]],
       device='cuda:0'), 'base_model.model.decoder.block.20.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0309, -0.0433, -0.0072,  ...,  0.0526, -0.0267,  0.0111],
        [ 0.0322,  0.0130, -0.0075,  ..., -0.0538,  0.0289,  0.0092],
        [ 0.0119,  0.0142,  0.0159,  ..., -0.0186,  0.0124,  0.0052],
        ...,
        [-0.0182, -0.0393,  0.0251,  ...,  0.0051, -0.0294,  0.0166],
        [ 0.0204, -0.0257,  0.0171,  ...,  0.0160,  0.0026,  0.0258],
        [-0.0115, -0.0061,  0.0026,  ..., -0.0053,  0.0131,  0.0137]],
       device='cuda:0'), 'base_model.model.decoder.block.20.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0007, -0.0135, -0.0022,  ..., -0.0014,  0.0102, -0.0091],
        [-0.0013, -0.0089, -0.0008,  ..., -0.0011,  0.0063, -0.0077],
        [ 0.0135, -0.0018, -0.0098,  ...,  0.0087,  0.0182, -0.0043],
        ...,
        [ 0.0196, -0.0158, -0.0186,  ...,  0.0112,  0.0094, -0.0163],
        [ 0.0239, -0.0141, -0.0202,  ...,  0.0215,  0.0194, -0.0229],
        [-0.0105,  0.0090,  0.0092,  ..., -0.0076, -0.0030,  0.0103]],
       device='cuda:0'), 'base_model.model.decoder.block.20.layer.1.EncDecAttention.v.lora_A.weight': tensor([[ 0.0300, -0.0317, -0.0016,  ...,  0.0477, -0.0341, -0.0443],
        [ 0.0130, -0.0124,  0.0146,  ...,  0.0177,  0.0036,  0.0013],
        [-0.0037, -0.0089, -0.0270,  ..., -0.0676,  0.0243,  0.0449],
        ...,
        [ 0.0045,  0.0037, -0.0003,  ..., -0.0588,  0.0432,  0.0416],
        [ 0.0265, -0.0409, -0.0213,  ...,  0.0781, -0.0029, -0.0608],
        [-0.0193, -0.0034,  0.0216,  ..., -0.0217,  0.0456,  0.0479]],
       device='cuda:0'), 'base_model.model.decoder.block.20.layer.1.EncDecAttention.v.lora_B.weight': tensor([[-0.0056, -0.0064,  0.0067,  ..., -0.0014, -0.0009,  0.0006],
        [-0.0254, -0.0245,  0.0249,  ...,  0.0180, -0.0263,  0.0243],
        [-0.0236, -0.0116,  0.0207,  ...,  0.0163, -0.0246,  0.0150],
        ...,
        [ 0.0120,  0.0035, -0.0083,  ..., -0.0079,  0.0130, -0.0055],
        [ 0.0174,  0.0187, -0.0193,  ..., -0.0140,  0.0158, -0.0161],
        [ 0.0092,  0.0099, -0.0160,  ..., -0.0007,  0.0068, -0.0113]],
       device='cuda:0'), 'base_model.model.decoder.block.21.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0343,  0.0204,  0.0275,  ...,  0.0053, -0.0517, -0.0413],
        [-0.0248, -0.0211,  0.0076,  ...,  0.0244,  0.0250,  0.0150],
        [-0.0068, -0.0234,  0.0406,  ...,  0.0005, -0.0413, -0.0170],
        ...,
        [ 0.0428, -0.0168, -0.0036,  ..., -0.0061, -0.0196,  0.0055],
        [-0.0289,  0.0417, -0.0210,  ...,  0.0379,  0.0039,  0.0105],
        [ 0.0070,  0.0048,  0.0043,  ..., -0.0240, -0.0300, -0.0242]],
       device='cuda:0'), 'base_model.model.decoder.block.21.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 4.4348e-03, -2.6485e-03,  2.1192e-03,  ...,  6.5235e-04,
         -4.2449e-03,  2.7613e-03],
        [ 1.5684e-03, -2.8341e-04,  4.5689e-05,  ..., -1.3167e-03,
         -6.2685e-04, -4.6340e-04],
        [ 5.4143e-04,  6.6562e-04, -9.4752e-04,  ..., -2.5828e-03,
          7.1928e-04, -1.4943e-03],
        ...,
        [ 3.1052e-03, -1.0300e-02,  9.1300e-04,  ...,  5.9527e-03,
          1.4728e-03,  2.2390e-03],
        [-1.0343e-02,  3.8973e-03, -1.0227e-02,  ..., -2.1103e-03,
          1.1451e-02, -1.0909e-02],
        [ 9.7546e-03, -4.5729e-03,  9.8230e-03,  ...,  6.4971e-03,
         -9.7390e-03,  9.7719e-03]], device='cuda:0'), 'base_model.model.decoder.block.21.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0403, -0.0040, -0.0013,  ..., -0.0024,  0.0256,  0.0428],
        [-0.0482,  0.0095, -0.0140,  ..., -0.0013,  0.0166, -0.0315],
        [ 0.0045,  0.0172,  0.0354,  ..., -0.0425, -0.0294,  0.0075],
        ...,
        [-0.0317, -0.0092, -0.0335,  ...,  0.0219,  0.0336, -0.0158],
        [-0.0078, -0.0031, -0.0020,  ..., -0.0177, -0.0036, -0.0421],
        [ 0.0143, -0.0200, -0.0190,  ...,  0.0009, -0.0299,  0.0199]],
       device='cuda:0'), 'base_model.model.decoder.block.21.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0263, -0.0255,  0.0235,  ..., -0.0238, -0.0280,  0.0286],
        [ 0.0201, -0.0229,  0.0209,  ..., -0.0200, -0.0184,  0.0127],
        [-0.0039,  0.0015, -0.0030,  ..., -0.0007,  0.0009, -0.0090],
        ...,
        [ 0.0110, -0.0022,  0.0084,  ..., -0.0041,  0.0051,  0.0059],
        [ 0.0110, -0.0111,  0.0048,  ..., -0.0118, -0.0143,  0.0013],
        [ 0.0178, -0.0232,  0.0145,  ..., -0.0203, -0.0183,  0.0250]],
       device='cuda:0'), 'base_model.model.decoder.block.21.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0286, -0.0409, -0.0219,  ...,  0.0076,  0.0002, -0.0041],
        [-0.0026,  0.0309,  0.0021,  ..., -0.0009, -0.0211,  0.0268],
        [ 0.0109,  0.0248, -0.0301,  ...,  0.0485, -0.0336, -0.0063],
        ...,
        [-0.0150, -0.0146, -0.0259,  ..., -0.0196,  0.0254, -0.0009],
        [ 0.0092, -0.0122, -0.0444,  ...,  0.0018, -0.0069,  0.0313],
        [-0.0075, -0.0570, -0.0182,  ...,  0.0037,  0.0248, -0.0176]],
       device='cuda:0'), 'base_model.model.decoder.block.21.layer.1.EncDecAttention.q.lora_B.weight': tensor([[-0.0168,  0.0255,  0.0151,  ..., -0.0183, -0.0177, -0.0194],
        [-0.0206,  0.0197,  0.0183,  ..., -0.0195, -0.0225, -0.0275],
        [ 0.0361, -0.0352, -0.0348,  ...,  0.0332,  0.0327,  0.0342],
        ...,
        [ 0.0194, -0.0234, -0.0214,  ...,  0.0162,  0.0178,  0.0205],
        [-0.0017,  0.0059,  0.0040,  ..., -0.0048, -0.0056, -0.0058],
        [ 0.0077, -0.0114, -0.0011,  ...,  0.0101,  0.0192,  0.0077]],
       device='cuda:0'), 'base_model.model.decoder.block.21.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-3.5695e-02,  1.6790e-02,  5.7587e-04,  ..., -1.6495e-02,
          1.2606e-02,  4.8337e-02],
        [ 2.3041e-02, -3.3336e-02, -2.2698e-02,  ...,  5.9429e-02,
         -3.1289e-02, -4.4992e-02],
        [-1.5568e-02, -5.2893e-02, -1.2566e-02,  ...,  4.0882e-02,
          1.0086e-02, -9.9746e-03],
        ...,
        [ 2.4377e-02, -3.7222e-02,  1.3223e-02,  ...,  5.4595e-02,
         -3.3970e-02, -1.3829e-03],
        [ 1.6691e-02,  5.6453e-02, -1.8042e-02,  ..., -7.6591e-02,
          9.8997e-05,  4.6439e-02],
        [-9.6895e-03, -2.9741e-02,  1.2167e-02,  ...,  2.4567e-02,
          9.3858e-03, -3.2569e-02]], device='cuda:0'), 'base_model.model.decoder.block.21.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 0.0311, -0.0313, -0.0311,  ..., -0.0332,  0.0247, -0.0304],
        [ 0.0029,  0.0026,  0.0029,  ...,  0.0035, -0.0013, -0.0060],
        [-0.0258,  0.0229,  0.0264,  ...,  0.0237, -0.0149,  0.0358],
        ...,
        [ 0.0278, -0.0260, -0.0260,  ..., -0.0240,  0.0291, -0.0316],
        [ 0.0034,  0.0013, -0.0023,  ..., -0.0007, -0.0021, -0.0052],
        [ 0.0103, -0.0093, -0.0117,  ..., -0.0091,  0.0129, -0.0151]],
       device='cuda:0'), 'base_model.model.decoder.block.22.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0075,  0.0101,  0.0066,  ...,  0.0137, -0.0290, -0.0332],
        [-0.0080, -0.0053, -0.0135,  ...,  0.0092, -0.0176, -0.0109],
        [-0.0237, -0.0107,  0.0388,  ..., -0.0160, -0.0025, -0.0372],
        ...,
        [ 0.0404, -0.0182,  0.0073,  ...,  0.0007,  0.0072, -0.0113],
        [ 0.0124, -0.0111,  0.0153,  ..., -0.0410, -0.0116, -0.0214],
        [-0.0053, -0.0287, -0.0165,  ..., -0.0356,  0.0135, -0.0157]],
       device='cuda:0'), 'base_model.model.decoder.block.22.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0063,  0.0048,  0.0027,  ...,  0.0076, -0.0064, -0.0074],
        [ 0.0014, -0.0117, -0.0015,  ..., -0.0004,  0.0117,  0.0018],
        [ 0.0156, -0.0110,  0.0123,  ..., -0.0135,  0.0156,  0.0141],
        ...,
        [ 0.0047, -0.0093,  0.0169,  ..., -0.0119,  0.0120,  0.0156],
        [-0.0021,  0.0090, -0.0040,  ...,  0.0050, -0.0092, -0.0039],
        [-0.0010, -0.0057, -0.0066,  ..., -0.0030,  0.0066, -0.0035]],
       device='cuda:0'), 'base_model.model.decoder.block.22.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0456, -0.0016, -0.0174,  ...,  0.0192, -0.0071, -0.0623],
        [ 0.0340,  0.0215, -0.0414,  ..., -0.0354, -0.0016, -0.0324],
        [ 0.0472, -0.0280, -0.0585,  ..., -0.0303, -0.0207, -0.0495],
        ...,
        [-0.0485, -0.0319, -0.0091,  ...,  0.0348,  0.0049,  0.0335],
        [-0.0007,  0.0192, -0.0118,  ..., -0.0202, -0.0005, -0.0519],
        [-0.0050, -0.0187,  0.0160,  ...,  0.0365,  0.0291,  0.0377]],
       device='cuda:0'), 'base_model.model.decoder.block.22.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0233, -0.0288, -0.0115,  ...,  0.0231, -0.0130,  0.0227],
        [-0.0201, -0.0285, -0.0253,  ...,  0.0275, -0.0260,  0.0299],
        [-0.0058, -0.0212, -0.0120,  ...,  0.0103, -0.0118,  0.0128],
        ...,
        [ 0.0127,  0.0124,  0.0190,  ..., -0.0125,  0.0157, -0.0204],
        [-0.0180, -0.0212, -0.0175,  ...,  0.0185, -0.0135,  0.0195],
        [-0.0200, -0.0104, -0.0042,  ...,  0.0166, -0.0149,  0.0107]],
       device='cuda:0'), 'base_model.model.decoder.block.22.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0084,  0.0150, -0.0145,  ..., -0.0232, -0.0257, -0.0149],
        [ 0.0048, -0.0269,  0.0020,  ..., -0.0380, -0.0280,  0.0107],
        [-0.0511,  0.0117, -0.0309,  ..., -0.0178, -0.0143,  0.0124],
        ...,
        [ 0.0064,  0.0060,  0.0122,  ...,  0.0043, -0.0113,  0.0389],
        [-0.0138,  0.0269,  0.0226,  ...,  0.0071, -0.0021, -0.0120],
        [-0.0171,  0.0366, -0.0063,  ...,  0.0260,  0.0141, -0.0228]],
       device='cuda:0'), 'base_model.model.decoder.block.22.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0172,  0.0150, -0.0139,  ..., -0.0159, -0.0151, -0.0160],
        [ 0.0052,  0.0058,  0.0026,  ..., -0.0062, -0.0036, -0.0068],
        [ 0.0252,  0.0213, -0.0235,  ..., -0.0217, -0.0223, -0.0177],
        ...,
        [-0.0061, -0.0087,  0.0110,  ...,  0.0117,  0.0126,  0.0129],
        [-0.0169, -0.0116,  0.0146,  ...,  0.0165,  0.0159,  0.0200],
        [ 0.0153,  0.0157, -0.0118,  ..., -0.0148, -0.0138, -0.0143]],
       device='cuda:0'), 'base_model.model.decoder.block.22.layer.1.EncDecAttention.v.lora_A.weight': tensor([[ 0.0323,  0.0615, -0.0286,  ..., -0.0212,  0.0202,  0.0314],
        [-0.0259,  0.0106, -0.0415,  ..., -0.0654, -0.0317,  0.0154],
        [-0.0052, -0.0413,  0.0448,  ..., -0.0346,  0.0133, -0.0400],
        ...,
        [ 0.0140,  0.0323, -0.0152,  ..., -0.0569, -0.0013,  0.0601],
        [-0.0142, -0.0644,  0.0147,  ...,  0.0293, -0.0159, -0.0258],
        [ 0.0235,  0.0580, -0.0371,  ..., -0.0377, -0.0346,  0.0627]],
       device='cuda:0'), 'base_model.model.decoder.block.22.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 0.0026,  0.0069,  0.0095,  ..., -0.0012, -0.0045, -0.0018],
        [ 0.0145,  0.0194,  0.0195,  ...,  0.0182, -0.0195,  0.0177],
        [ 0.0069,  0.0107,  0.0172,  ...,  0.0084, -0.0127,  0.0161],
        ...,
        [ 0.0442,  0.0403, -0.0388,  ...,  0.0492, -0.0602,  0.0562],
        [-0.0228, -0.0222,  0.0388,  ..., -0.0255,  0.0293, -0.0299],
        [-0.0342, -0.0308,  0.0272,  ..., -0.0349,  0.0436, -0.0529]],
       device='cuda:0'), 'base_model.model.decoder.block.23.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0229,  0.0198, -0.0208,  ..., -0.0067,  0.0257, -0.0064],
        [-0.0030, -0.0127,  0.0058,  ..., -0.0117,  0.0089,  0.0005],
        [ 0.0391,  0.0234, -0.0144,  ...,  0.0433,  0.0134,  0.0189],
        ...,
        [-0.0255, -0.0331,  0.0175,  ..., -0.0176, -0.0498, -0.0518],
        [ 0.0129,  0.0078, -0.0142,  ..., -0.0030,  0.0345, -0.0058],
        [-0.0361, -0.0408,  0.0383,  ..., -0.0299, -0.0016, -0.0264]],
       device='cuda:0'), 'base_model.model.decoder.block.23.layer.0.SelfAttention.q.lora_B.weight': tensor([[-9.2302e-03,  4.7082e-04, -1.6876e-03,  ...,  1.5358e-03,
          3.6666e-04,  2.2397e-03],
        [ 6.2450e-03, -8.7633e-05,  1.2780e-03,  ..., -2.2387e-03,
         -7.6455e-03, -3.6240e-03],
        [ 7.5522e-03,  2.7108e-03,  4.8920e-03,  ..., -3.7219e-03,
          1.2247e-03, -5.4539e-03],
        ...,
        [ 8.3448e-03,  3.1519e-03,  2.2037e-03,  ...,  1.5882e-03,
          2.2505e-03, -1.0635e-03],
        [ 3.2382e-03,  8.2808e-03,  8.4877e-03,  ..., -5.4786e-03,
          9.0481e-03, -9.2424e-03],
        [-3.7960e-03, -1.2958e-02, -1.1950e-02,  ...,  9.1770e-03,
         -1.1545e-02,  7.7533e-03]], device='cuda:0'), 'base_model.model.decoder.block.23.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0188,  0.0154,  0.0271,  ..., -0.0192,  0.0154,  0.0396],
        [ 0.0480,  0.0190, -0.0210,  ...,  0.0008,  0.0105, -0.0560],
        [-0.0263, -0.0329,  0.0212,  ..., -0.0381,  0.0282,  0.0525],
        ...,
        [-0.0505,  0.0039,  0.0447,  ..., -0.0066, -0.0126,  0.0477],
        [-0.0024, -0.0017,  0.0043,  ..., -0.0188, -0.0137,  0.0257],
        [-0.0486,  0.0265,  0.0539,  ..., -0.0064,  0.0338,  0.0425]],
       device='cuda:0'), 'base_model.model.decoder.block.23.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0036, -0.0072,  0.0104,  ...,  0.0092,  0.0054,  0.0088],
        [ 0.0053, -0.0083,  0.0139,  ...,  0.0087,  0.0064,  0.0107],
        [-0.0054,  0.0058, -0.0056,  ..., -0.0039, -0.0077, -0.0046],
        ...,
        [ 0.0016, -0.0062,  0.0073,  ...,  0.0109,  0.0018,  0.0105],
        [-0.0155,  0.0177, -0.0152,  ..., -0.0161, -0.0173, -0.0173],
        [ 0.0070, -0.0056, -0.0022,  ...,  0.0054,  0.0060, -0.0005]],
       device='cuda:0'), 'base_model.model.decoder.block.23.layer.1.EncDecAttention.q.lora_A.weight': tensor([[ 0.0013, -0.0205, -0.0237,  ..., -0.0034,  0.0223, -0.0525],
        [-0.0195, -0.0254, -0.0189,  ...,  0.0134,  0.0543, -0.0611],
        [ 0.0078,  0.0246, -0.0145,  ...,  0.0244,  0.0562, -0.0179],
        ...,
        [-0.0164, -0.0022, -0.0291,  ..., -0.0274, -0.0185, -0.0195],
        [ 0.0101,  0.0265,  0.0126,  ...,  0.0248,  0.0240, -0.0185],
        [-0.0226, -0.0008, -0.0094,  ...,  0.0020, -0.0359, -0.0004]],
       device='cuda:0'), 'base_model.model.decoder.block.23.layer.1.EncDecAttention.q.lora_B.weight': tensor([[-1.3781e-02, -1.4603e-02, -1.4217e-02,  ..., -9.6741e-03,
         -1.7460e-02,  1.1197e-02],
        [ 2.7672e-03,  5.4598e-03, -8.9388e-05,  ...,  2.6880e-03,
          1.1293e-02, -9.0429e-03],
        [ 3.1708e-03, -5.6693e-04, -4.4513e-03,  ...,  1.0293e-03,
          1.9967e-03, -4.6050e-03],
        ...,
        [-1.9154e-02,  2.7257e-03, -2.3589e-03,  ...,  1.8899e-03,
          6.3871e-03, -4.6289e-03],
        [ 1.3242e-03,  1.7306e-03, -3.5451e-03,  ..., -3.5201e-03,
         -5.5426e-03,  4.4722e-03],
        [ 1.3347e-02, -6.4381e-03, -1.2333e-02,  ..., -5.1522e-03,
         -1.4272e-02,  1.4489e-02]], device='cuda:0'), 'base_model.model.decoder.block.23.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0109, -0.0416,  0.0268,  ...,  0.0437, -0.0096, -0.0408],
        [-0.0006, -0.0368, -0.0175,  ...,  0.0711, -0.0297, -0.0360],
        [-0.0097, -0.0302, -0.0101,  ...,  0.0609, -0.0160, -0.0315],
        ...,
        [-0.0348,  0.0250,  0.0112,  ..., -0.0480,  0.0094,  0.0005],
        [-0.0217, -0.0358, -0.0137,  ...,  0.0188, -0.0225, -0.0686],
        [-0.0249, -0.0334, -0.0092,  ...,  0.0849, -0.0285, -0.0345]],
       device='cuda:0'), 'base_model.model.decoder.block.23.layer.1.EncDecAttention.v.lora_B.weight': tensor([[-0.0244, -0.0160, -0.0185,  ..., -0.0134, -0.0150, -0.0029],
        [-0.0028,  0.0118,  0.0127,  ...,  0.0023,  0.0139,  0.0067],
        [ 0.0218,  0.0264,  0.0261,  ...,  0.0093,  0.0255,  0.0199],
        ...,
        [-0.0049, -0.0083, -0.0071,  ..., -0.0327, -0.0097,  0.0223],
        [ 0.0283,  0.0257,  0.0250,  ...,  0.0135,  0.0239, -0.0075],
        [ 0.0004, -0.0003, -0.0002,  ..., -0.0289, -0.0049,  0.0312]],
       device='cuda:0')}

In [18]:

!du -h $checkpoint_name

19M	financial_sentiment_analysis_lora_v1.pt

In [ ]: