diff --git a/examples/pet_lora_seq2seq.ipynb b/examples/pet_lora_seq2seq.ipynb index f8d86c6..3f02675 100644 --- a/examples/pet_lora_seq2seq.ipynb +++ b/examples/pet_lora_seq2seq.ipynb @@ -41,2211 +41,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "8d0850ac", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "trainable params: 4718592 || all params: 1234299904 || trainable%: 0.38228893842642636\n" - ] - }, - { - "data": { - "text/plain": [ - "PETModelForSeq2SeqLM(\n", - " (base_model): LoRAModel(\n", - " (model): MT5ForConditionalGeneration(\n", - " (shared): Embedding(250112, 1024)\n", - " (encoder): T5Stack(\n", - " (embed_tokens): Embedding(250112, 1024)\n", - " (block): ModuleList(\n", - " (0): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (relative_attention_bias): Embedding(32, 16)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (1): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (2): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (3): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (4): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (5): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (6): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (7): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (8): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (9): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (10): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (11): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (12): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (13): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (14): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (15): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (16): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (17): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (18): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (19): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (20): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (21): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (22): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (23): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " )\n", - " (final_layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (decoder): T5Stack(\n", - " (embed_tokens): Embedding(250112, 1024)\n", - " (block): ModuleList(\n", - " (0): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (relative_attention_bias): Embedding(32, 16)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (1): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (2): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (3): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (4): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (5): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (6): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (7): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (8): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (9): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (10): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (11): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (12): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (13): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (14): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (15): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (16): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (17): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (18): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (19): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (20): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (21): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (22): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " (23): T5Block(\n", - " (layer): ModuleList(\n", - " (0): T5LayerSelfAttention(\n", - " (SelfAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (1): T5LayerCrossAttention(\n", - " (EncDecAttention): T5Attention(\n", - " (q): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (k): Linear(in_features=1024, out_features=1024, bias=False)\n", - " (v): Linear(\n", - " in_features=1024, out_features=1024, bias=False\n", - " (lora_dropout): Dropout(p=0.1, inplace=False)\n", - " (lora_A): Linear(in_features=1024, out_features=16, bias=False)\n", - " (lora_B): Linear(in_features=16, out_features=1024, bias=False)\n", - " )\n", - " (o): Linear(in_features=1024, out_features=1024, bias=False)\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (2): T5LayerFF(\n", - " (DenseReluDense): T5DenseGatedActDense(\n", - " (wi_0): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wi_1): Linear(in_features=1024, out_features=2816, bias=False)\n", - " (wo): Linear(in_features=2816, out_features=1024, bias=False)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " (act): NewGELUActivation()\n", - " )\n", - " (layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " )\n", - " (final_layer_norm): FusedRMSNorm(torch.Size([1024]), eps=1e-06, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (lm_head): Linear(in_features=1024, out_features=250112, bias=False)\n", - " )\n", - " )\n", - ")" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# creating model\n", "pet_config = get_pet_config(config)\n", @@ -2549,2209 +348,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "a8de6005", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'base_model.model.encoder.block.0.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0011, 0.0126, -0.0392, ..., 0.0117, -0.0080, 0.0227],\n", - " [ 0.0239, -0.0152, 0.0174, ..., -0.0209, -0.0093, -0.0250],\n", - " [ 0.0197, 0.0023, 0.0045, ..., -0.0073, -0.0106, -0.0038],\n", - " ...,\n", - " [-0.0116, 0.0015, 0.0146, ..., 0.0017, -0.0237, -0.0077],\n", - " [ 0.0143, 0.0330, 0.0059, ..., 0.0124, 0.0045, -0.0180],\n", - " [ 0.0095, 0.0024, 0.0352, ..., 0.0064, 0.0230, -0.0452]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.0.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0161, -0.0037, 0.0014, ..., 0.0039, 0.0235, -0.0148],\n", - " [-0.0086, -0.0180, 0.0253, ..., 0.0277, 0.0029, -0.0193],\n", - " [ 0.0004, 0.0172, 0.0078, ..., 0.0094, -0.0130, 0.0012],\n", - " ...,\n", - " [ 0.0010, 0.0024, 0.0202, ..., 0.0259, -0.0117, 0.0107],\n", - " [ 0.0256, 0.0210, -0.0293, ..., -0.0210, 0.0059, -0.0250],\n", - " [-0.0056, -0.0048, 0.0187, ..., 0.0086, -0.0178, 0.0021]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.0.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0205, -0.0127, 0.0213, ..., -0.0042, 0.0034, 0.0243],\n", - " [ 0.0197, -0.0426, -0.0326, ..., 0.0069, -0.0203, 0.0358],\n", - " [-0.0216, 0.0100, -0.0084, ..., 0.0446, -0.0540, 0.0452],\n", - " ...,\n", - " [ 0.0578, -0.0072, 0.0047, ..., -0.0078, -0.0231, 0.0116],\n", - " [ 0.0131, 0.0072, 0.0372, ..., -0.0183, -0.0214, -0.0139],\n", - " [ 0.0013, 0.0402, -0.0062, ..., 0.0365, 0.0286, -0.0329]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.0.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0046, -0.0044, 0.0061, ..., -0.0013, -0.0303, -0.0034],\n", - " [ 0.0060, 0.0068, -0.0019, ..., -0.0046, -0.0336, -0.0189],\n", - " [-0.0039, -0.0053, 0.0089, ..., -0.0128, 0.0059, -0.0216],\n", - " ...,\n", - " [ 0.0052, -0.0199, -0.0261, ..., -0.0253, 0.0173, -0.0027],\n", - " [ 0.0098, -0.0081, -0.0105, ..., -0.0074, -0.0016, -0.0026],\n", - " [ 0.0134, -0.0060, 0.0122, ..., 0.0021, -0.0014, 0.0148]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.1.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0306, -0.0243, -0.0414, ..., 0.0121, 0.0280, 0.0006],\n", - " [-0.0425, -0.0292, 0.0076, ..., 0.0322, -0.0244, 0.0346],\n", - " [-0.0081, 0.0155, -0.0289, ..., -0.0078, 0.0121, -0.0135],\n", - " ...,\n", - " [ 0.0161, 0.0303, -0.0498, ..., -0.0148, 0.0022, -0.0115],\n", - " [ 0.0205, 0.0197, -0.0203, ..., -0.0424, -0.0252, -0.0418],\n", - " [ 0.0098, -0.0085, -0.0391, ..., 0.0195, -0.0180, -0.0565]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.1.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 1.3011e-02, -8.4894e-03, 1.4550e-02, ..., -2.8281e-02,\n", - " -1.5803e-02, -6.7014e-03],\n", - " [-6.9723e-03, 2.5044e-02, 1.0645e-03, ..., -1.9179e-02,\n", - " -2.0764e-02, -1.5413e-02],\n", - " [ 1.1783e-02, -1.5737e-02, 1.4881e-02, ..., -3.9572e-03,\n", - " 1.2995e-02, 2.8500e-02],\n", - " ...,\n", - " [-1.0119e-02, 3.6680e-03, 2.4997e-03, ..., -1.1717e-02,\n", - " 6.6043e-03, -3.2334e-03],\n", - " [-1.1863e-02, -6.9381e-04, -1.1733e-02, ..., -3.1885e-03,\n", - " -1.4473e-02, 1.3692e-03],\n", - " [-7.2537e-03, 2.4926e-02, 2.9916e-03, ..., -6.9848e-04,\n", - " -5.9038e-03, -5.7989e-05]], device='cuda:0'), 'base_model.model.encoder.block.1.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0199, -0.0045, 0.0448, ..., -0.0153, -0.0625, -0.0284],\n", - " [-0.0109, -0.0260, -0.0110, ..., 0.0036, 0.0067, -0.0112],\n", - " [ 0.0537, 0.0035, -0.0319, ..., -0.0041, -0.0150, 0.0522],\n", - " ...,\n", - " [ 0.0215, 0.0431, -0.0165, ..., 0.0086, 0.0072, 0.0050],\n", - " [ 0.0327, -0.0188, 0.0122, ..., 0.0043, -0.0070, -0.0060],\n", - " [ 0.0123, -0.0429, 0.0311, ..., -0.0028, 0.0158, -0.0010]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.1.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0024, -0.0146, -0.0314, ..., 0.0111, -0.0105, -0.0042],\n", - " [ 0.0165, 0.0012, -0.0041, ..., 0.0039, -0.0018, 0.0084],\n", - " [-0.0278, 0.0229, -0.0050, ..., -0.0019, -0.0090, 0.0025],\n", - " ...,\n", - " [ 0.0228, 0.0125, -0.0167, ..., -0.0163, -0.0046, 0.0134],\n", - " [-0.0191, -0.0080, 0.0129, ..., 0.0015, -0.0096, -0.0112],\n", - " [ 0.0306, -0.0115, -0.0116, ..., 0.0047, -0.0097, -0.0013]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.2.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0183, -0.0045, -0.0373, ..., 0.0028, -0.0018, 0.0099],\n", - " [-0.0376, 0.0284, -0.0107, ..., -0.0196, -0.0107, 0.0114],\n", - " [ 0.0004, 0.0118, -0.0360, ..., -0.0594, 0.0541, 0.0094],\n", - " ...,\n", - " [ 0.0039, -0.0022, 0.0018, ..., 0.0467, 0.0007, -0.0348],\n", - " [-0.0036, 0.0106, 0.0272, ..., 0.0413, 0.0270, 0.0300],\n", - " [-0.0325, -0.0090, 0.0226, ..., -0.0035, 0.0270, -0.0215]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.2.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0052, 0.0233, -0.0136, ..., -0.0086, 0.0216, 0.0165],\n", - " [ 0.0174, 0.0217, 0.0273, ..., -0.0157, 0.0139, 0.0167],\n", - " [ 0.0038, -0.0342, 0.0208, ..., 0.0044, -0.0041, 0.0206],\n", - " ...,\n", - " [-0.0028, 0.0279, 0.0068, ..., -0.0180, 0.0110, 0.0245],\n", - " [-0.0083, 0.0002, 0.0032, ..., 0.0041, 0.0075, 0.0023],\n", - " [ 0.0368, -0.0012, -0.0103, ..., -0.0077, -0.0096, -0.0221]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.2.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0620, -0.0185, -0.0250, ..., -0.0078, 0.0048, 0.0218],\n", - " [ 0.0750, -0.0434, 0.0007, ..., -0.0083, 0.0214, 0.0416],\n", - " [ 0.0181, -0.0213, 0.0101, ..., 0.0022, 0.0036, 0.0140],\n", - " ...,\n", - " [-0.0114, 0.0142, -0.0239, ..., -0.0055, 0.0138, 0.0047],\n", - " [-0.0246, -0.0030, 0.0016, ..., 0.0156, -0.0082, -0.0074],\n", - " [-0.0391, 0.0287, -0.0008, ..., -0.0037, -0.0228, -0.0337]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.2.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0043, 0.0104, 0.0072, ..., -0.0040, 0.0189, -0.0040],\n", - " [ 0.0302, 0.0278, 0.0314, ..., -0.0275, 0.0137, -0.0292],\n", - " [-0.0450, -0.0463, -0.0416, ..., 0.0435, -0.0480, 0.0439],\n", - " ...,\n", - " [-0.0043, -0.0020, 0.0152, ..., -0.0201, 0.0059, -0.0025],\n", - " [-0.0169, -0.0239, -0.0168, ..., 0.0162, 0.0110, 0.0223],\n", - " [ 0.0274, 0.0228, 0.0166, ..., -0.0190, -0.0184, -0.0188]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.3.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0117, -0.0470, -0.0126, ..., 0.0142, -0.0365, -0.0033],\n", - " [ 0.0198, 0.0106, 0.0006, ..., -0.0219, 0.0049, 0.0288],\n", - " [ 0.0024, 0.0177, 0.0296, ..., -0.0142, 0.0018, -0.0186],\n", - " ...,\n", - " [-0.0298, -0.0051, 0.0024, ..., 0.0243, -0.0158, 0.0213],\n", - " [-0.0088, -0.0265, 0.0339, ..., -0.0116, -0.0214, -0.0225],\n", - " [-0.0259, -0.0074, 0.0242, ..., 0.0003, -0.0149, 0.0445]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.3.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0196, 0.0119, -0.0112, ..., 0.0255, -0.0357, -0.0064],\n", - " [-0.0150, 0.0146, 0.0108, ..., -0.0005, -0.0003, 0.0082],\n", - " [-0.0081, -0.0125, 0.0033, ..., -0.0002, 0.0076, -0.0156],\n", - " ...,\n", - " [-0.0096, 0.0218, -0.0031, ..., 0.0057, -0.0008, 0.0058],\n", - " [ 0.0084, -0.0011, 0.0069, ..., -0.0263, 0.0075, -0.0034],\n", - " [ 0.0040, 0.0189, 0.0171, ..., -0.0186, 0.0084, 0.0118]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.3.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0178, 0.0288, 0.0192, ..., 0.0078, -0.0146, -0.0158],\n", - " [-0.0173, 0.0058, -0.0253, ..., -0.0172, -0.0134, -0.0071],\n", - " [-0.0167, -0.0333, -0.0169, ..., -0.0249, -0.0090, 0.0195],\n", - " ...,\n", - " [ 0.0093, -0.0083, 0.0031, ..., 0.0143, 0.0326, 0.0302],\n", - " [-0.0282, 0.0131, 0.0045, ..., -0.0152, 0.0099, -0.0038],\n", - " [-0.0027, -0.0236, -0.0395, ..., -0.0288, 0.0060, -0.0122]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.3.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0041, 0.0012, 0.0171, ..., 0.0103, -0.0087, -0.0037],\n", - " [ 0.0061, -0.0059, 0.0004, ..., 0.0228, -0.0394, 0.0151],\n", - " [-0.0123, -0.0029, 0.0168, ..., 0.0022, -0.0009, 0.0042],\n", - " ...,\n", - " [-0.0318, 0.0018, 0.0162, ..., -0.0191, 0.0094, 0.0015],\n", - " [-0.0176, 0.0092, 0.0165, ..., -0.0086, 0.0150, 0.0036],\n", - " [ 0.0359, 0.0136, -0.0198, ..., 0.0134, -0.0206, -0.0085]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.4.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0325, 0.0371, 0.0161, ..., -0.0039, -0.0107, -0.0066],\n", - " [-0.0440, -0.0042, 0.0123, ..., -0.0205, -0.0390, 0.0127],\n", - " [ 0.0115, -0.0246, -0.0256, ..., -0.0113, 0.0181, -0.0191],\n", - " ...,\n", - " [-0.0033, 0.0258, 0.0214, ..., 0.0249, -0.0544, 0.0062],\n", - " [-0.0294, -0.0302, -0.0316, ..., -0.0032, -0.0195, 0.0247],\n", - " [-0.0197, -0.0115, 0.0108, ..., 0.0308, -0.0469, -0.0307]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.4.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0078, -0.0045, 0.0053, ..., 0.0212, 0.0162, -0.0151],\n", - " [-0.0228, 0.0114, 0.0248, ..., 0.0080, 0.0110, -0.0130],\n", - " [-0.0307, 0.0002, 0.0129, ..., -0.0120, -0.0139, -0.0271],\n", - " ...,\n", - " [-0.0056, -0.0221, 0.0047, ..., 0.0003, 0.0098, -0.0024],\n", - " [ 0.0210, 0.0042, 0.0005, ..., 0.0077, -0.0075, 0.0073],\n", - " [ 0.0255, 0.0454, 0.0152, ..., 0.0398, 0.0391, -0.0117]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.4.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0198, -0.0339, 0.0026, ..., 0.0093, 0.0370, 0.0019],\n", - " [-0.0242, -0.0109, 0.0368, ..., -0.0427, -0.0158, -0.0527],\n", - " [-0.0196, 0.0233, 0.0298, ..., 0.0180, -0.0326, 0.0129],\n", - " ...,\n", - " [ 0.0229, 0.0284, -0.0155, ..., 0.0192, 0.0025, 0.0028],\n", - " [ 0.0174, -0.0308, 0.0193, ..., 0.0195, -0.0141, 0.0278],\n", - " [ 0.0147, 0.0028, -0.0325, ..., -0.0281, 0.0010, 0.0087]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.4.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0090, 0.0005, -0.0142, ..., -0.0017, -0.0026, -0.0015],\n", - " [ 0.0117, -0.0037, 0.0098, ..., -0.0013, 0.0009, -0.0046],\n", - " [-0.0163, -0.0004, -0.0039, ..., 0.0007, 0.0109, 0.0066],\n", - " ...,\n", - " [-0.0187, 0.0198, 0.0076, ..., 0.0136, 0.0249, -0.0021],\n", - " [ 0.0008, 0.0045, -0.0036, ..., -0.0057, 0.0173, 0.0004],\n", - " [-0.0046, -0.0078, -0.0104, ..., -0.0017, -0.0123, 0.0134]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.5.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 4.6854e-02, 2.2295e-02, 6.4651e-03, ..., 2.5293e-02,\n", - " 2.1630e-02, 1.2340e-02],\n", - " [ 1.3335e-02, 3.8377e-02, -4.0418e-02, ..., -1.1633e-02,\n", - " -1.8374e-02, -1.1622e-02],\n", - " [-5.3741e-02, 7.6245e-03, 5.9155e-02, ..., -1.7577e-02,\n", - " -4.4678e-02, -1.1400e-02],\n", - " ...,\n", - " [-3.7519e-02, 7.7807e-03, 2.1634e-02, ..., -1.2836e-02,\n", - " -3.3665e-02, -3.3249e-05],\n", - " [ 8.0464e-03, -1.4498e-03, 1.2061e-02, ..., 2.6527e-02,\n", - " 2.2661e-03, 1.5226e-02],\n", - " [ 1.3127e-02, 3.4243e-02, -7.6076e-03, ..., -1.7030e-02,\n", - " 3.7331e-02, -2.0033e-02]], device='cuda:0'), 'base_model.model.encoder.block.5.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0211, 0.0119, 0.0196, ..., 0.0310, 0.0261, 0.0212],\n", - " [ 0.0150, 0.0073, -0.0072, ..., -0.0296, -0.0019, -0.0055],\n", - " [ 0.0040, -0.0227, -0.0023, ..., -0.0025, -0.0282, -0.0286],\n", - " ...,\n", - " [-0.0339, 0.0068, 0.0066, ..., 0.0262, 0.0112, 0.0062],\n", - " [ 0.0111, 0.0063, -0.0170, ..., -0.0230, -0.0087, 0.0053],\n", - " [-0.0182, 0.0041, 0.0143, ..., 0.0118, 0.0134, -0.0034]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.5.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0077, -0.0111, 0.0294, ..., -0.0395, 0.0136, 0.0262],\n", - " [-0.0040, -0.0357, -0.0582, ..., -0.0002, -0.0220, -0.0344],\n", - " [ 0.0358, 0.0259, 0.0419, ..., 0.0121, 0.0241, -0.0243],\n", - " ...,\n", - " [ 0.0032, -0.0115, -0.0457, ..., 0.0290, 0.0213, -0.0060],\n", - " [ 0.0011, 0.0236, 0.0375, ..., 0.0009, 0.0075, 0.0216],\n", - " [-0.0085, 0.0274, 0.0160, ..., 0.0069, -0.0044, 0.0034]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.5.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0093, 0.0031, 0.0132, ..., -0.0114, 0.0090, 0.0082],\n", - " [ 0.0036, 0.0175, -0.0117, ..., 0.0118, -0.0122, -0.0122],\n", - " [-0.0102, 0.0050, 0.0037, ..., 0.0009, -0.0026, -0.0070],\n", - " ...,\n", - " [ 0.0107, -0.0064, 0.0131, ..., -0.0112, 0.0124, 0.0116],\n", - " [-0.0078, 0.0084, 0.0060, ..., 0.0016, 0.0031, -0.0009],\n", - " [ 0.0077, 0.0074, -0.0095, ..., 0.0063, -0.0104, -0.0140]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.6.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0052, -0.0190, -0.0309, ..., 0.0368, 0.0119, 0.0045],\n", - " [ 0.0246, 0.0021, -0.0215, ..., 0.0109, -0.0025, -0.0697],\n", - " [-0.0161, 0.0301, -0.0043, ..., -0.0135, -0.0248, -0.0280],\n", - " ...,\n", - " [ 0.0573, -0.0103, 0.0023, ..., 0.0115, 0.0555, -0.0128],\n", - " [-0.0021, 0.0424, -0.0038, ..., 0.0515, 0.0286, 0.0078],\n", - " [-0.0103, -0.0092, 0.0028, ..., 0.0229, -0.0282, -0.0122]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.6.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0185, 0.0261, -0.0120, ..., 0.0008, 0.0034, -0.0148],\n", - " [-0.0174, 0.0061, -0.0011, ..., -0.0079, 0.0253, 0.0086],\n", - " [ 0.0133, -0.0025, -0.0073, ..., -0.0178, 0.0182, 0.0027],\n", - " ...,\n", - " [ 0.0103, -0.0069, -0.0184, ..., -0.0013, 0.0038, -0.0002],\n", - " [ 0.0133, 0.0024, 0.0006, ..., -0.0015, -0.0018, 0.0221],\n", - " [ 0.0162, -0.0035, -0.0131, ..., -0.0185, -0.0240, 0.0108]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.6.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0123, -0.0133, -0.0479, ..., 0.0003, 0.0049, -0.0105],\n", - " [-0.0214, 0.0090, 0.0156, ..., -0.0335, -0.0089, -0.0161],\n", - " [ 0.0283, -0.0100, 0.0151, ..., -0.0126, -0.0030, -0.0421],\n", - " ...,\n", - " [ 0.0319, 0.0161, -0.0138, ..., 0.0119, 0.0346, 0.0224],\n", - " [ 0.0075, 0.0111, 0.0164, ..., 0.0003, 0.0058, 0.0415],\n", - " [ 0.0122, 0.0280, -0.0096, ..., 0.0193, 0.0315, 0.0293]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.6.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0163, 0.0121, 0.0083, ..., -0.0032, -0.0093, 0.0055],\n", - " [ 0.0157, -0.0087, -0.0007, ..., 0.0037, 0.0028, 0.0137],\n", - " [ 0.0043, -0.0024, -0.0150, ..., 0.0119, 0.0082, 0.0046],\n", - " ...,\n", - " [ 0.0077, -0.0089, -0.0054, ..., 0.0100, 0.0072, 0.0122],\n", - " [ 0.0042, -0.0140, 0.0039, ..., 0.0058, -0.0015, -0.0027],\n", - " [ 0.0118, -0.0002, -0.0154, ..., -0.0005, -0.0107, -0.0160]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.7.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0249, -0.0245, 0.0263, ..., 0.0231, 0.0002, -0.0161],\n", - " [-0.0284, 0.0376, -0.0152, ..., 0.0037, -0.0273, -0.0441],\n", - " [ 0.0552, -0.0126, 0.0192, ..., 0.0180, 0.0117, -0.0234],\n", - " ...,\n", - " [ 0.0255, -0.0163, 0.0113, ..., -0.0212, -0.0115, 0.0077],\n", - " [-0.0227, -0.0103, -0.0073, ..., 0.0213, -0.0048, -0.0216],\n", - " [ 0.0301, -0.0359, -0.0198, ..., 0.0291, 0.0122, 0.0169]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.7.layer.0.SelfAttention.q.lora_B.weight': tensor([[-3.6794e-03, -1.1577e-02, -6.4594e-03, ..., -5.8575e-03,\n", - " 6.4691e-03, 6.1957e-03],\n", - " [-2.4289e-02, 4.5173e-03, -2.1506e-02, ..., -2.8095e-02,\n", - " 4.1102e-02, -1.3792e-02],\n", - " [ 7.2917e-03, -2.7127e-02, 6.7668e-05, ..., 7.6069e-03,\n", - " -3.3562e-03, 7.5219e-03],\n", - " ...,\n", - " [-4.4854e-03, -2.3905e-03, -6.4449e-03, ..., -7.4162e-03,\n", - " 9.0684e-03, 2.0679e-04],\n", - " [-4.0559e-03, 1.3454e-02, -5.7965e-03, ..., 4.8874e-04,\n", - " 1.2333e-02, -8.3721e-03],\n", - " [-3.8800e-03, -1.0870e-02, -1.6102e-02, ..., -2.2499e-02,\n", - " 1.5781e-02, 1.3765e-02]], device='cuda:0'), 'base_model.model.encoder.block.7.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0115, 0.0034, 0.0256, ..., -0.0181, -0.0146, -0.0219],\n", - " [ 0.0442, -0.0186, -0.0151, ..., -0.0228, 0.0146, 0.0321],\n", - " [-0.0081, -0.0252, 0.0188, ..., -0.0297, 0.0230, 0.0064],\n", - " ...,\n", - " [ 0.0098, 0.0081, 0.0102, ..., -0.0116, -0.0435, 0.0061],\n", - " [-0.0069, 0.0040, -0.0126, ..., -0.0046, -0.0336, 0.0175],\n", - " [ 0.0045, -0.0129, 0.0252, ..., -0.0328, -0.0095, -0.0250]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.7.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0095, -0.0049, 0.0090, ..., -0.0012, -0.0114, 0.0152],\n", - " [-0.0096, -0.0163, 0.0093, ..., -0.0169, -0.0110, 0.0143],\n", - " [ 0.0172, 0.0033, -0.0209, ..., 0.0011, 0.0101, -0.0149],\n", - " ...,\n", - " [-0.0223, -0.0201, 0.0178, ..., 0.0135, -0.0135, -0.0085],\n", - " [ 0.0029, -0.0050, -0.0139, ..., -0.0083, 0.0040, -0.0070],\n", - " [-0.0176, -0.0048, 0.0106, ..., 0.0100, -0.0008, 0.0078]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.8.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0547, -0.0251, -0.0336, ..., 0.0047, -0.0150, -0.0334],\n", - " [ 0.0297, 0.0428, -0.0070, ..., 0.0102, 0.0042, -0.0159],\n", - " [-0.0193, 0.0235, 0.0417, ..., -0.0160, -0.0147, 0.0220],\n", - " ...,\n", - " [ 0.0081, -0.0281, 0.0176, ..., -0.0033, 0.0308, 0.0319],\n", - " [ 0.0157, 0.0331, -0.0299, ..., 0.0307, 0.0073, 0.0037],\n", - " [-0.0328, -0.0115, -0.0006, ..., -0.0026, -0.0101, 0.0336]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.8.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0221, 0.0148, 0.0164, ..., -0.0353, -0.0054, -0.0331],\n", - " [ 0.0102, 0.0060, -0.0084, ..., 0.0259, 0.0239, 0.0233],\n", - " [-0.0187, -0.0233, -0.0057, ..., -0.0188, 0.0055, 0.0035],\n", - " ...,\n", - " [ 0.0039, -0.0079, -0.0280, ..., 0.0096, 0.0152, 0.0069],\n", - " [-0.0040, 0.0051, 0.0182, ..., -0.0046, -0.0015, 0.0005],\n", - " [-0.0031, 0.0172, -0.0042, ..., -0.0034, 0.0112, 0.0120]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.8.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0472, 0.0169, -0.0520, ..., -0.0187, 0.0038, 0.0221],\n", - " [-0.0078, -0.0035, 0.0112, ..., -0.0010, 0.0342, 0.0090],\n", - " [-0.0125, 0.0102, 0.0478, ..., -0.0135, -0.0045, 0.0193],\n", - " ...,\n", - " [-0.0448, -0.0266, 0.0023, ..., 0.0034, 0.0374, 0.0349],\n", - " [ 0.0378, 0.0289, -0.0119, ..., -0.0113, 0.0009, -0.0055],\n", - " [ 0.0265, -0.0061, -0.0120, ..., 0.0267, -0.0213, -0.0053]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.8.layer.0.SelfAttention.v.lora_B.weight': tensor([[-1.1674e-02, -1.4416e-02, 1.6741e-02, ..., -2.0069e-02,\n", - " 1.3848e-02, -9.4953e-05],\n", - " [ 2.8010e-03, -6.6800e-03, -1.4637e-03, ..., -1.3755e-02,\n", - " 1.9308e-02, 1.2938e-02],\n", - " [ 1.5742e-02, -7.6401e-03, -1.9585e-02, ..., -7.9043e-03,\n", - " 9.8119e-03, -3.4114e-03],\n", - " ...,\n", - " [ 8.7199e-03, -1.3911e-02, 1.9556e-02, ..., -1.1287e-02,\n", - " 1.3224e-02, -1.2388e-02],\n", - " [-1.4302e-02, 2.5201e-03, 1.5830e-02, ..., 1.1704e-02,\n", - " -9.5458e-03, -2.3148e-03],\n", - " [-1.2765e-02, -1.4806e-02, 3.0562e-02, ..., -1.1230e-03,\n", - " -2.1326e-03, 1.5665e-02]], device='cuda:0'), 'base_model.model.encoder.block.9.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0424, 0.0046, -0.0188, ..., -0.0201, 0.0062, -0.0091],\n", - " [ 0.0254, 0.0364, 0.0047, ..., -0.0357, -0.0316, -0.0028],\n", - " [-0.0442, 0.0029, -0.0119, ..., -0.0363, -0.0380, -0.0143],\n", - " ...,\n", - " [ 0.0478, 0.0122, 0.0339, ..., 0.0345, -0.0044, -0.0131],\n", - " [ 0.0319, 0.0082, -0.0122, ..., 0.0015, -0.0245, -0.0195],\n", - " [ 0.0207, 0.0255, 0.0040, ..., -0.0058, -0.0114, -0.0195]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.9.layer.0.SelfAttention.q.lora_B.weight': tensor([[-2.2374e-03, 5.3732e-03, 1.9213e-02, ..., 1.1581e-02,\n", - " -3.2477e-03, -4.9648e-04],\n", - " [ 7.1751e-03, -8.7029e-03, 3.1709e-02, ..., -2.3060e-02,\n", - " -1.1236e-02, 1.3812e-02],\n", - " [ 4.9562e-04, -4.9533e-03, -4.9173e-03, ..., 2.0552e-02,\n", - " -1.9678e-03, -3.8285e-05],\n", - " ...,\n", - " [ 1.4587e-02, -2.4020e-02, -1.1420e-02, ..., 1.3391e-03,\n", - " -4.4101e-03, 3.2557e-03],\n", - " [-2.1797e-02, 3.1295e-02, 2.9931e-02, ..., 1.0324e-02,\n", - " 2.1327e-02, -1.1652e-02],\n", - " [-2.7132e-02, 5.5430e-03, -4.0748e-02, ..., 7.3987e-03,\n", - " 1.2751e-02, -1.0383e-02]], device='cuda:0'), 'base_model.model.encoder.block.9.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0212, 0.0094, -0.0345, ..., -0.0055, 0.0172, 0.0463],\n", - " [-0.0218, 0.0271, 0.0145, ..., -0.0182, -0.0750, 0.0068],\n", - " [-0.0287, -0.0087, 0.0458, ..., 0.0255, 0.0004, -0.0237],\n", - " ...,\n", - " [-0.0278, 0.0398, 0.0445, ..., 0.0031, -0.0069, -0.0153],\n", - " [ 0.0143, -0.0057, 0.0280, ..., 0.0183, -0.0014, 0.0398],\n", - " [-0.0347, -0.0115, 0.0330, ..., 0.0281, -0.0414, -0.0309]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.9.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0175, -0.0150, -0.0187, ..., -0.0194, -0.0235, -0.0187],\n", - " [-0.0096, 0.0072, 0.0135, ..., 0.0082, 0.0054, 0.0180],\n", - " [-0.0190, 0.0146, 0.0120, ..., 0.0130, 0.0177, 0.0227],\n", - " ...,\n", - " [-0.0145, -0.0077, 0.0004, ..., 0.0100, -0.0152, 0.0171],\n", - " [ 0.0154, -0.0045, -0.0016, ..., -0.0170, 0.0122, -0.0131],\n", - " [-0.0150, -0.0046, -0.0022, ..., 0.0112, -0.0164, 0.0145]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.10.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0063, -0.0355, 0.0021, ..., 0.0098, 0.0202, -0.0121],\n", - " [-0.0238, 0.0037, 0.0345, ..., -0.0088, -0.0315, -0.0167],\n", - " [ 0.0099, -0.0115, -0.0161, ..., -0.0287, 0.0219, 0.0324],\n", - " ...,\n", - " [ 0.0052, -0.0403, -0.0258, ..., -0.0323, -0.0180, 0.0049],\n", - " [ 0.0460, -0.0125, 0.0313, ..., 0.0127, 0.0384, -0.0161],\n", - " [ 0.0363, -0.0080, -0.0062, ..., 0.0132, 0.0386, -0.0175]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.10.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0137, -0.0086, -0.0250, ..., -0.0072, -0.0013, 0.0127],\n", - " [ 0.0238, 0.0095, -0.0219, ..., 0.0205, 0.0050, 0.0102],\n", - " [ 0.0054, 0.0194, -0.0351, ..., -0.0171, 0.0097, 0.0261],\n", - " ...,\n", - " [-0.0058, -0.0221, 0.0088, ..., -0.0028, 0.0017, -0.0202],\n", - " [ 0.0044, -0.0364, 0.0107, ..., -0.0187, -0.0172, -0.0214],\n", - " [-0.0165, -0.0024, 0.0094, ..., -0.0130, -0.0179, -0.0093]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.10.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0242, -0.0273, 0.0012, ..., -0.0284, -0.0053, -0.0016],\n", - " [ 0.0436, -0.0018, -0.0349, ..., 0.0164, 0.0422, 0.0017],\n", - " [ 0.0485, 0.0392, 0.0135, ..., 0.0272, -0.0033, -0.0040],\n", - " ...,\n", - " [ 0.0098, 0.0046, -0.0287, ..., -0.0284, 0.0120, 0.0047],\n", - " [ 0.0139, -0.0017, -0.0226, ..., -0.0269, -0.0447, 0.0190],\n", - " [ 0.0453, -0.0130, -0.0016, ..., 0.0023, 0.0084, 0.0260]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.10.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0052, -0.0034, 0.0169, ..., -0.0010, 0.0101, 0.0065],\n", - " [-0.0235, 0.0224, -0.0020, ..., 0.0105, -0.0222, -0.0214],\n", - " [-0.0252, 0.0259, -0.0122, ..., 0.0191, -0.0154, -0.0236],\n", - " ...,\n", - " [-0.0093, 0.0063, 0.0104, ..., 0.0207, -0.0164, -0.0046],\n", - " [-0.0115, 0.0047, -0.0123, ..., 0.0146, -0.0010, -0.0085],\n", - " [-0.0081, 0.0099, 0.0202, ..., 0.0214, -0.0252, 0.0029]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.11.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0178, -0.0056, 0.0114, ..., 0.0069, 0.0038, 0.0309],\n", - " [ 0.0363, 0.0298, 0.0199, ..., -0.0196, -0.0228, 0.0145],\n", - " [-0.0566, 0.0105, -0.0255, ..., -0.0212, 0.0047, 0.0039],\n", - " ...,\n", - " [ 0.0267, 0.0304, -0.0092, ..., 0.0117, -0.0274, -0.0169],\n", - " [ 0.0403, 0.0225, -0.0204, ..., -0.0277, 0.0124, -0.0060],\n", - " [ 0.0238, -0.0275, 0.0223, ..., 0.0041, -0.0117, 0.0226]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.11.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 1.5251e-02, -1.3405e-02, -1.0952e-02, ..., -3.2804e-02,\n", - " 8.0364e-03, 1.1897e-02],\n", - " [-1.2090e-02, 1.9948e-02, 2.2969e-03, ..., 1.9390e-02,\n", - " 5.5107e-03, 8.3068e-03],\n", - " [-9.2829e-03, 1.3713e-02, 2.2380e-02, ..., -3.9204e-03,\n", - " -1.3160e-02, -3.6896e-03],\n", - " ...,\n", - " [-8.0247e-03, 6.4117e-03, 1.9469e-02, ..., 1.0616e-02,\n", - " 2.5570e-04, 2.6831e-03],\n", - " [-5.6667e-03, -5.0435e-03, -1.5369e-02, ..., -1.1559e-02,\n", - " 1.9381e-02, -7.5156e-05],\n", - " [ 5.1334e-03, 9.1880e-03, -2.8864e-02, ..., -2.4764e-02,\n", - " 1.5606e-02, -2.5908e-02]], device='cuda:0'), 'base_model.model.encoder.block.11.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0133, -0.0197, -0.0365, ..., -0.0401, -0.0523, 0.0024],\n", - " [-0.0132, 0.0168, 0.0084, ..., 0.0424, -0.0226, 0.0277],\n", - " [ 0.0298, 0.0316, -0.0208, ..., 0.0038, 0.0429, 0.0068],\n", - " ...,\n", - " [ 0.0310, 0.0084, 0.0016, ..., 0.0036, 0.0146, -0.0063],\n", - " [-0.0516, -0.0046, -0.0126, ..., 0.0213, -0.0387, -0.0145],\n", - " [-0.0594, -0.0328, -0.0431, ..., -0.0225, -0.0079, -0.0229]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.11.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0250, -0.0376, 0.0345, ..., 0.0266, -0.0233, -0.0127],\n", - " [-0.0047, 0.0099, -0.0125, ..., -0.0094, 0.0040, 0.0016],\n", - " [-0.0066, 0.0164, -0.0167, ..., -0.0113, 0.0158, 0.0106],\n", - " ...,\n", - " [-0.0111, 0.0109, -0.0093, ..., -0.0007, 0.0101, -0.0061],\n", - " [-0.0109, 0.0009, -0.0044, ..., -0.0191, -0.0032, 0.0205],\n", - " [-0.0100, 0.0034, -0.0044, ..., -0.0122, 0.0104, 0.0291]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.12.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0140, 0.0012, -0.0323, ..., 0.0490, -0.0234, -0.0137],\n", - " [-0.0366, -0.0191, 0.0091, ..., 0.0381, 0.0512, -0.0114],\n", - " [ 0.0090, -0.0214, -0.0278, ..., 0.0384, 0.0307, -0.0009],\n", - " ...,\n", - " [-0.0556, 0.0305, 0.0202, ..., 0.0185, -0.0310, -0.0131],\n", - " [-0.0287, 0.0159, 0.0026, ..., 0.0202, 0.0368, 0.0184],\n", - " [-0.0384, 0.0130, -0.0241, ..., 0.0191, 0.0524, 0.0031]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.12.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0059, 0.0123, 0.0067, ..., 0.0095, 0.0090, 0.0237],\n", - " [ 0.0024, 0.0133, 0.0313, ..., 0.0079, 0.0124, 0.0065],\n", - " [ 0.0086, -0.0328, -0.0109, ..., -0.0251, -0.0013, -0.0018],\n", - " ...,\n", - " [-0.0003, -0.0192, 0.0292, ..., -0.0260, 0.0006, -0.0082],\n", - " [ 0.0058, -0.0187, -0.0136, ..., -0.0056, -0.0078, -0.0284],\n", - " [ 0.0213, -0.0099, -0.0119, ..., 0.0173, 0.0065, -0.0007]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.12.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0155, 0.0088, 0.0113, ..., 0.0075, 0.0049, 0.0154],\n", - " [-0.0268, 0.0203, 0.0017, ..., -0.0269, 0.0272, 0.0151],\n", - " [-0.0063, 0.0055, -0.0013, ..., -0.0193, 0.0099, -0.0143],\n", - " ...,\n", - " [ 0.0315, 0.0117, -0.0101, ..., 0.0034, 0.0140, 0.0136],\n", - " [ 0.0042, -0.0548, -0.0144, ..., 0.0041, 0.0296, 0.0246],\n", - " [ 0.0081, 0.0255, 0.0194, ..., 0.0020, 0.0124, 0.0246]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.12.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0034, 0.0007, -0.0139, ..., -0.0021, 0.0076, 0.0024],\n", - " [-0.0063, -0.0191, -0.0012, ..., 0.0197, 0.0059, 0.0199],\n", - " [ 0.0098, 0.0127, -0.0077, ..., -0.0152, -0.0014, -0.0179],\n", - " ...,\n", - " [-0.0012, -0.0055, 0.0102, ..., 0.0071, 0.0157, 0.0007],\n", - " [-0.0223, -0.0310, -0.0248, ..., 0.0279, 0.0295, 0.0272],\n", - " [-0.0144, -0.0010, -0.0043, ..., 0.0101, 0.0027, 0.0096]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.13.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0528, 0.0265, -0.0006, ..., 0.0108, 0.0268, -0.0358],\n", - " [-0.0148, -0.0179, 0.0211, ..., -0.0519, -0.0323, -0.0162],\n", - " [-0.0082, -0.0202, -0.0363, ..., -0.0067, 0.0422, 0.0203],\n", - " ...,\n", - " [ 0.0119, 0.0068, 0.0021, ..., -0.0284, -0.0126, -0.0077],\n", - " [-0.0252, 0.0406, -0.0096, ..., 0.0380, -0.0298, -0.0124],\n", - " [ 0.0174, -0.0255, 0.0100, ..., -0.0062, -0.0038, -0.0483]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.13.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0046, 0.0007, 0.0040, ..., 0.0127, -0.0086, 0.0045],\n", - " [-0.0034, 0.0012, -0.0269, ..., 0.0077, 0.0077, 0.0079],\n", - " [-0.0255, 0.0091, 0.0121, ..., 0.0027, -0.0190, 0.0250],\n", - " ...,\n", - " [-0.0013, -0.0045, 0.0094, ..., -0.0040, -0.0030, 0.0165],\n", - " [-0.0094, -0.0013, 0.0098, ..., 0.0043, -0.0060, 0.0057],\n", - " [-0.0247, 0.0240, 0.0263, ..., 0.0149, -0.0302, 0.0101]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.13.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0235, 0.0104, 0.0366, ..., 0.0335, 0.0118, -0.0104],\n", - " [-0.0169, 0.0107, -0.0293, ..., -0.0196, 0.0248, 0.0136],\n", - " [ 0.0046, 0.0114, 0.0013, ..., -0.0061, 0.0208, 0.0137],\n", - " ...,\n", - " [-0.0008, 0.0332, 0.0251, ..., -0.0024, 0.0390, -0.0130],\n", - " [ 0.0226, -0.0223, -0.0490, ..., -0.0123, -0.0358, 0.0141],\n", - " [-0.0114, -0.0098, -0.0323, ..., 0.0350, -0.0451, 0.0060]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.13.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 1.3388e-03, -7.8071e-03, -4.4231e-03, ..., 6.8468e-03,\n", - " -8.6865e-03, -6.6189e-03],\n", - " [-8.4138e-03, 3.9207e-03, -1.7379e-02, ..., -1.4644e-02,\n", - " 2.0210e-02, 1.8017e-02],\n", - " [-8.6932e-03, 3.4929e-03, -8.5747e-03, ..., -7.9256e-05,\n", - " -7.3968e-03, -1.2425e-02],\n", - " ...,\n", - " [ 2.5251e-02, -1.5215e-02, 1.2540e-02, ..., 5.3171e-04,\n", - " 5.1631e-03, -6.5597e-04],\n", - " [-4.4899e-03, -1.2352e-02, -3.8436e-03, ..., 4.8080e-03,\n", - " -9.7884e-04, -6.4385e-03],\n", - " [ 1.2952e-02, -6.3606e-03, 6.3772e-03, ..., -9.4017e-03,\n", - " 5.4179e-03, 6.1516e-03]], device='cuda:0'), 'base_model.model.encoder.block.14.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0031, -0.0145, -0.0498, ..., 0.0126, -0.0046, 0.0666],\n", - " [-0.0352, 0.0089, 0.0178, ..., -0.0369, -0.0284, -0.0033],\n", - " [-0.0229, 0.0080, -0.0020, ..., 0.0681, 0.0048, 0.0553],\n", - " ...,\n", - " [ 0.0118, 0.0116, -0.0289, ..., 0.0028, -0.0169, 0.0415],\n", - " [ 0.0137, -0.0024, 0.0268, ..., 0.0435, 0.0123, -0.0126],\n", - " [ 0.0422, 0.0337, 0.0022, ..., 0.0373, 0.0318, 0.0344]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.14.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0116, -0.0045, 0.0094, ..., 0.0165, 0.0021, 0.0011],\n", - " [-0.0034, 0.0033, -0.0199, ..., 0.0003, 0.0146, 0.0141],\n", - " [-0.0055, 0.0080, -0.0076, ..., -0.0197, -0.0134, -0.0145],\n", - " ...,\n", - " [ 0.0130, -0.0213, 0.0119, ..., 0.0132, 0.0252, 0.0126],\n", - " [ 0.0011, -0.0145, 0.0209, ..., -0.0021, 0.0018, -0.0081],\n", - " [-0.0271, -0.0154, 0.0062, ..., 0.0095, 0.0025, -0.0112]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.14.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0097, -0.0242, 0.0057, ..., 0.0214, -0.0358, -0.0250],\n", - " [-0.0355, -0.0014, -0.0570, ..., 0.0239, 0.0224, -0.0252],\n", - " [-0.0124, 0.0291, 0.0428, ..., -0.0319, 0.0109, 0.0128],\n", - " ...,\n", - " [ 0.0115, -0.0210, -0.0035, ..., 0.0227, -0.0120, 0.0143],\n", - " [-0.0309, 0.0038, 0.0248, ..., 0.0052, -0.0469, 0.0364],\n", - " [-0.0103, 0.0015, -0.0015, ..., -0.0410, 0.0415, -0.0185]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.14.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0389, -0.0082, 0.0113, ..., -0.0270, 0.0057, 0.0258],\n", - " [ 0.0010, 0.0043, -0.0075, ..., -0.0001, -0.0128, -0.0028],\n", - " [ 0.0128, 0.0135, -0.0123, ..., 0.0104, -0.0072, -0.0109],\n", - " ...,\n", - " [-0.0414, -0.0318, 0.0169, ..., -0.0431, 0.0356, 0.0451],\n", - " [ 0.0091, -0.0132, 0.0102, ..., -0.0135, 0.0111, -0.0035],\n", - " [ 0.0005, 0.0068, -0.0088, ..., -0.0002, 0.0083, -0.0006]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.15.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0069, 0.0003, -0.0256, ..., 0.0258, -0.0335, 0.0484],\n", - " [-0.0216, -0.0322, 0.0204, ..., 0.0170, -0.0222, -0.0108],\n", - " [ 0.0008, 0.0105, -0.0380, ..., -0.0099, -0.0394, 0.0262],\n", - " ...,\n", - " [-0.0015, -0.0016, -0.0252, ..., -0.0252, -0.0368, -0.0343],\n", - " [-0.0253, 0.0274, 0.0133, ..., 0.0494, -0.0482, 0.0473],\n", - " [-0.0317, 0.0028, 0.0004, ..., 0.0471, 0.0196, 0.0111]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.15.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0266, -0.0135, 0.0290, ..., -0.0318, -0.0019, 0.0112],\n", - " [-0.0016, 0.0150, -0.0023, ..., 0.0111, -0.0071, -0.0280],\n", - " [-0.0286, 0.0020, -0.0203, ..., 0.0246, -0.0086, -0.0261],\n", - " ...,\n", - " [-0.0133, -0.0072, -0.0033, ..., 0.0133, -0.0128, -0.0120],\n", - " [-0.0055, -0.0159, -0.0040, ..., 0.0039, -0.0175, -0.0134],\n", - " [-0.0024, 0.0035, 0.0067, ..., 0.0042, -0.0094, -0.0038]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.15.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 1.7631e-03, 3.6335e-02, -2.2382e-02, ..., -1.4592e-02,\n", - " 5.8360e-02, -1.8392e-02],\n", - " [ 1.4876e-02, 2.0355e-02, -2.3993e-02, ..., 1.8249e-02,\n", - " -2.0012e-03, -2.4304e-02],\n", - " [-1.7222e-02, 8.3348e-04, -2.3049e-02, ..., -2.6813e-02,\n", - " 3.8505e-02, -1.4321e-02],\n", - " ...,\n", - " [-1.3301e-02, -1.7976e-05, -1.0527e-02, ..., 4.7702e-02,\n", - " 2.2238e-03, -2.4194e-02],\n", - " [-2.4366e-02, -3.5398e-02, 2.0618e-02, ..., 3.4942e-03,\n", - " -4.9058e-02, 8.0171e-03],\n", - " [-3.0048e-03, 3.7638e-02, -3.8622e-03, ..., -5.0610e-03,\n", - " 5.2482e-02, -1.0379e-02]], device='cuda:0'), 'base_model.model.encoder.block.15.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 2.5725e-03, 3.5926e-03, 1.9242e-03, ..., -1.1365e-02,\n", - " -2.6424e-04, -4.3521e-03],\n", - " [-2.1243e-03, -1.0515e-02, 4.6888e-03, ..., -3.1430e-03,\n", - " -6.3100e-03, -6.4766e-03],\n", - " [-1.0521e-02, -1.0788e-02, -6.9606e-03, ..., 1.1522e-02,\n", - " 3.2694e-03, -1.3453e-02],\n", - " ...,\n", - " [-3.5505e-05, -2.4789e-02, 5.3724e-03, ..., 1.2047e-02,\n", - " -5.5704e-03, -1.3944e-02],\n", - " [-1.4204e-02, -2.3716e-02, -2.5142e-03, ..., 1.1288e-02,\n", - " 1.0336e-02, -1.5918e-02],\n", - " [-9.6600e-03, -4.7291e-04, -1.8597e-02, ..., 1.7493e-02,\n", - " 1.8787e-02, -1.6483e-02]], device='cuda:0'), 'base_model.model.encoder.block.16.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0099, -0.0129, 0.0517, ..., -0.0158, 0.0089, 0.0313],\n", - " [ 0.0373, -0.0277, 0.0096, ..., -0.0250, -0.0250, 0.0282],\n", - " [-0.0122, -0.0129, -0.0159, ..., -0.0003, 0.0164, 0.0209],\n", - " ...,\n", - " [ 0.0360, -0.0043, 0.0127, ..., 0.0088, -0.0333, -0.0175],\n", - " [ 0.0071, -0.0533, -0.0338, ..., 0.0417, -0.0052, 0.0208],\n", - " [ 0.0142, -0.0381, 0.0059, ..., -0.0328, 0.0163, 0.0182]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.16.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0117, -0.0307, -0.0021, ..., 0.0123, -0.0253, -0.0016],\n", - " [-0.0085, 0.0046, 0.0009, ..., -0.0039, 0.0105, -0.0178],\n", - " [ 0.0037, 0.0178, 0.0198, ..., -0.0141, 0.0087, 0.0110],\n", - " ...,\n", - " [-0.0101, 0.0057, 0.0065, ..., 0.0019, 0.0080, -0.0162],\n", - " [ 0.0104, 0.0138, 0.0215, ..., -0.0142, 0.0151, 0.0108],\n", - " [ 0.0186, 0.0051, 0.0312, ..., -0.0250, 0.0252, 0.0021]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.16.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0258, -0.0153, -0.0037, ..., -0.0136, -0.0433, 0.0465],\n", - " [-0.0139, -0.0489, -0.0567, ..., 0.0056, 0.0010, 0.0062],\n", - " [ 0.0086, 0.0541, 0.0437, ..., -0.0202, 0.0503, -0.0420],\n", - " ...,\n", - " [ 0.0304, 0.0422, -0.0691, ..., -0.0162, 0.0146, 0.0323],\n", - " [-0.0310, 0.0305, 0.0252, ..., 0.0340, -0.0040, -0.0321],\n", - " [ 0.0063, 0.0173, -0.0269, ..., 0.0041, -0.0490, 0.0162]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.16.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0372, -0.0004, -0.0395, ..., -0.0252, -0.0212, 0.0400],\n", - " [ 0.0557, 0.0030, -0.0577, ..., -0.0191, -0.0292, 0.0516],\n", - " [-0.0057, 0.0290, 0.0045, ..., 0.0348, -0.0138, -0.0093],\n", - " ...,\n", - " [-0.0273, -0.0235, 0.0251, ..., -0.0232, 0.0207, -0.0205],\n", - " [ 0.0374, 0.0416, -0.0401, ..., 0.0239, -0.0532, 0.0429],\n", - " [-0.0201, -0.0159, 0.0268, ..., -0.0145, 0.0342, -0.0210]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.17.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0124, 0.0274, 0.0223, ..., -0.0135, 0.0262, 0.0426],\n", - " [-0.0069, 0.0114, 0.0237, ..., 0.0522, 0.0042, 0.0107],\n", - " [ 0.0016, -0.0762, -0.0267, ..., 0.0016, 0.0416, -0.0379],\n", - " ...,\n", - " [-0.0107, -0.0051, 0.0061, ..., 0.0077, -0.0267, -0.0538],\n", - " [-0.0166, -0.0188, -0.0361, ..., -0.0321, 0.0282, -0.0272],\n", - " [ 0.0195, -0.0308, 0.0041, ..., -0.0343, -0.0432, -0.0337]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.17.layer.0.SelfAttention.q.lora_B.weight': tensor([[-1.4683e-02, -9.0315e-03, 1.3334e-02, ..., 6.3660e-03,\n", - " 3.7701e-03, 1.8145e-02],\n", - " [ 2.2848e-03, -4.4163e-03, 1.6257e-02, ..., 8.8848e-03,\n", - " 1.0144e-02, 6.2522e-03],\n", - " [ 1.5033e-02, 1.0294e-02, -1.7981e-02, ..., -8.9641e-03,\n", - " -2.1745e-02, -2.4771e-02],\n", - " ...,\n", - " [ 2.1272e-04, -7.2553e-03, -2.3124e-06, ..., 3.4841e-03,\n", - " -1.0348e-02, -2.2534e-02],\n", - " [-2.2342e-03, 1.8137e-02, -6.5606e-03, ..., 2.7981e-03,\n", - " -8.4023e-03, 1.4239e-02],\n", - " [-3.0212e-03, 3.7128e-03, -1.7892e-02, ..., 1.1220e-02,\n", - " -2.2024e-02, -8.1391e-03]], device='cuda:0'), 'base_model.model.encoder.block.17.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0019, -0.0137, -0.0012, ..., -0.0406, -0.0212, 0.0466],\n", - " [ 0.0227, -0.0154, 0.0074, ..., 0.0155, -0.0104, 0.0346],\n", - " [-0.0455, 0.0035, 0.0101, ..., -0.0231, -0.0183, -0.0431],\n", - " ...,\n", - " [-0.0447, 0.0181, 0.0305, ..., 0.0417, -0.0317, -0.0054],\n", - " [ 0.0427, -0.0357, -0.0128, ..., -0.0214, 0.0126, 0.0405],\n", - " [-0.0563, 0.0184, -0.0125, ..., 0.0406, -0.0077, -0.0211]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.17.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0090, -0.0032, -0.0055, ..., -0.0054, -0.0003, 0.0061],\n", - " [ 0.0058, 0.0085, -0.0087, ..., -0.0043, 0.0099, 0.0082],\n", - " [ 0.0087, 0.0208, -0.0196, ..., -0.0181, 0.0206, 0.0036],\n", - " ...,\n", - " [-0.0395, -0.0220, 0.0149, ..., 0.0223, -0.0172, 0.0448],\n", - " [-0.0227, -0.0071, 0.0050, ..., 0.0142, -0.0116, 0.0192],\n", - " [-0.0102, 0.0058, -0.0205, ..., -0.0027, 0.0044, 0.0175]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.18.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0241, 0.0187, 0.0110, ..., 0.0043, -0.0370, 0.0384],\n", - " [ 0.0038, 0.0020, -0.0249, ..., -0.0397, 0.0138, -0.0315],\n", - " [-0.0226, -0.0243, -0.0236, ..., 0.0053, -0.0367, -0.0319],\n", - " ...,\n", - " [ 0.0032, 0.0287, -0.0235, ..., -0.0466, -0.0053, 0.0106],\n", - " [-0.0264, -0.0175, -0.0500, ..., -0.0234, 0.0288, -0.0165],\n", - " [-0.0156, 0.0258, 0.0038, ..., -0.0575, -0.0257, -0.0088]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.18.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0184, -0.0091, -0.0198, ..., -0.0144, -0.0019, -0.0017],\n", - " [-0.0016, 0.0060, -0.0113, ..., 0.0096, -0.0067, 0.0017],\n", - " [-0.0023, -0.0006, -0.0505, ..., -0.0005, -0.0123, 0.0116],\n", - " ...,\n", - " [-0.0247, 0.0048, 0.0250, ..., 0.0066, -0.0144, 0.0116],\n", - " [ 0.0060, -0.0079, -0.0068, ..., -0.0094, 0.0171, -0.0095],\n", - " [-0.0086, -0.0147, 0.0030, ..., -0.0247, 0.0092, -0.0146]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.18.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0043, 0.0391, -0.0416, ..., -0.0416, 0.0277, 0.0104],\n", - " [-0.0030, 0.0658, -0.0103, ..., -0.0245, 0.0268, 0.0276],\n", - " [ 0.0047, 0.0221, -0.0067, ..., 0.0026, 0.0197, 0.0375],\n", - " ...,\n", - " [-0.0245, -0.0708, 0.0604, ..., 0.0216, 0.0002, -0.0149],\n", - " [ 0.0146, -0.0349, 0.0189, ..., -0.0136, -0.0235, 0.0123],\n", - " [-0.0083, -0.0331, 0.0403, ..., 0.0157, -0.0047, -0.0127]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.18.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0209, -0.0291, -0.0244, ..., 0.0253, 0.0127, -0.0248],\n", - " [ 0.0053, 0.0060, 0.0070, ..., -0.0051, -0.0021, -0.0103],\n", - " [ 0.0148, 0.0031, 0.0126, ..., -0.0117, -0.0017, -0.0147],\n", - " ...,\n", - " [ 0.0200, 0.0158, 0.0231, ..., -0.0149, -0.0121, 0.0300],\n", - " [-0.0008, -0.0023, -0.0031, ..., 0.0107, 0.0033, 0.0051],\n", - " [ 0.0240, 0.0208, 0.0184, ..., -0.0156, -0.0005, 0.0188]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.19.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0249, -0.0210, -0.0265, ..., 0.0108, -0.0111, -0.0171],\n", - " [-0.0351, 0.0088, -0.0190, ..., 0.0171, 0.0221, 0.0042],\n", - " [ 0.0030, 0.0014, -0.0108, ..., 0.0105, 0.0679, 0.0146],\n", - " ...,\n", - " [ 0.0347, -0.0230, 0.0132, ..., -0.0003, -0.0530, 0.0031],\n", - " [ 0.0138, 0.0345, 0.0108, ..., 0.0178, 0.0252, 0.0180],\n", - " [ 0.0034, 0.0017, -0.0196, ..., -0.0089, -0.0449, -0.0323]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.19.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0205, -0.0094, -0.0139, ..., 0.0070, -0.0126, 0.0226],\n", - " [-0.0010, -0.0070, -0.0134, ..., 0.0123, -0.0060, 0.0030],\n", - " [-0.0007, -0.0004, -0.0068, ..., 0.0005, 0.0003, 0.0037],\n", - " ...,\n", - " [ 0.0378, -0.0430, -0.0296, ..., 0.0429, -0.0451, 0.0347],\n", - " [-0.0320, 0.0106, 0.0203, ..., -0.0211, 0.0239, -0.0175],\n", - " [ 0.0210, -0.0117, -0.0150, ..., 0.0160, -0.0256, 0.0168]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.19.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 1.4258e-02, -2.0162e-02, 1.4808e-02, ..., 6.3350e-02,\n", - " 3.1473e-02, -2.1102e-02],\n", - " [ 3.0901e-03, -4.1341e-02, 2.0951e-02, ..., 1.8360e-02,\n", - " -6.7527e-03, 1.2380e-02],\n", - " [ 2.9749e-02, 5.5730e-05, -6.0016e-03, ..., 4.4938e-02,\n", - " 6.8638e-03, -1.5086e-02],\n", - " ...,\n", - " [-6.1253e-03, 1.3493e-03, 1.6801e-02, ..., 5.0317e-03,\n", - " -5.3136e-03, 1.7972e-02],\n", - " [ 1.9262e-02, -1.0739e-03, 5.0379e-02, ..., 6.5103e-02,\n", - " 2.6762e-02, -1.0854e-02],\n", - " [ 2.7448e-02, 2.8939e-03, 4.7538e-02, ..., 4.8242e-02,\n", - " 2.4452e-02, -2.7716e-02]], device='cuda:0'), 'base_model.model.encoder.block.19.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0130, -0.0170, -0.0146, ..., -0.0144, -0.0169, -0.0175],\n", - " [-0.0181, -0.0188, -0.0173, ..., -0.0193, -0.0192, -0.0150],\n", - " [ 0.0139, 0.0166, 0.0089, ..., 0.0140, 0.0107, 0.0067],\n", - " ...,\n", - " [-0.0119, -0.0102, -0.0134, ..., -0.0145, -0.0182, -0.0151],\n", - " [ 0.0106, 0.0094, 0.0102, ..., 0.0079, 0.0062, 0.0125],\n", - " [-0.0259, -0.0304, -0.0267, ..., -0.0262, -0.0288, -0.0286]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.20.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0136, -0.0010, -0.0112, ..., 0.0021, -0.0050, 0.0332],\n", - " [ 0.0083, -0.0270, -0.0023, ..., 0.0004, -0.0469, 0.0221],\n", - " [-0.0129, -0.0622, 0.0074, ..., 0.0102, -0.0384, 0.0262],\n", - " ...,\n", - " [-0.0207, -0.0472, 0.0020, ..., 0.0109, -0.0551, 0.0010],\n", - " [-0.0171, 0.0238, -0.0192, ..., 0.0388, 0.0184, -0.0068],\n", - " [ 0.0377, -0.0111, 0.0170, ..., -0.0148, -0.0103, 0.0415]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.20.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0201, -0.0002, -0.0038, ..., -0.0076, 0.0105, -0.0084],\n", - " [-0.0349, 0.0177, -0.0228, ..., 0.0044, 0.0148, -0.0253],\n", - " [-0.0230, 0.0061, 0.0187, ..., 0.0196, -0.0039, 0.0043],\n", - " ...,\n", - " [-0.0062, 0.0211, -0.0158, ..., -0.0114, 0.0234, -0.0168],\n", - " [-0.0158, -0.0012, 0.0284, ..., 0.0266, -0.0246, 0.0207],\n", - " [-0.0249, 0.0377, -0.0102, ..., 0.0059, 0.0032, -0.0116]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.20.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0300, 0.0260, 0.0026, ..., 0.0304, 0.0345, 0.0474],\n", - " [-0.0201, -0.0045, 0.0204, ..., -0.0125, -0.0221, -0.0285],\n", - " [-0.0180, -0.0343, 0.0173, ..., 0.0428, 0.0268, 0.0424],\n", - " ...,\n", - " [ 0.0010, -0.0441, -0.0271, ..., 0.0089, 0.0099, -0.0180],\n", - " [-0.0098, -0.0109, 0.0484, ..., 0.0118, 0.0068, -0.0187],\n", - " [-0.0206, -0.0263, -0.0238, ..., 0.0390, 0.0060, 0.0163]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.20.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0201, -0.0216, 0.0119, ..., 0.0183, -0.0170, 0.0124],\n", - " [ 0.0130, -0.0100, 0.0180, ..., 0.0223, 0.0051, 0.0060],\n", - " [-0.0278, 0.0233, -0.0264, ..., -0.0253, 0.0271, -0.0330],\n", - " ...,\n", - " [-0.0052, -0.0034, 0.0056, ..., 0.0010, 0.0099, 0.0036],\n", - " [-0.0003, -0.0029, -0.0112, ..., -0.0089, -0.0206, 0.0131],\n", - " [ 0.0036, -0.0245, 0.0003, ..., -0.0018, -0.0305, 0.0315]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.21.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0078, -0.0170, 0.0393, ..., -0.0284, 0.0189, 0.0169],\n", - " [-0.0003, 0.0058, -0.0220, ..., 0.0204, -0.0052, -0.0126],\n", - " [ 0.0164, 0.0175, -0.0063, ..., 0.0019, 0.0210, -0.0453],\n", - " ...,\n", - " [ 0.0046, 0.0172, 0.0201, ..., -0.0191, 0.0161, 0.0512],\n", - " [ 0.0214, -0.0041, 0.0220, ..., -0.0172, 0.0319, 0.0360],\n", - " [-0.0295, 0.0370, -0.0062, ..., 0.0288, 0.0298, 0.0034]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.21.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0021, 0.0006, 0.0097, ..., -0.0011, 0.0026, -0.0374],\n", - " [ 0.0022, -0.0127, -0.0116, ..., 0.0035, 0.0007, 0.0061],\n", - " [-0.0159, 0.0103, 0.0079, ..., -0.0159, -0.0147, -0.0258],\n", - " ...,\n", - " [-0.0059, 0.0005, -0.0015, ..., -0.0041, -0.0017, -0.0200],\n", - " [-0.0226, 0.0153, 0.0165, ..., -0.0202, -0.0170, 0.0057],\n", - " [-0.0090, 0.0059, 0.0104, ..., -0.0058, -0.0050, -0.0039]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.21.layer.0.SelfAttention.v.lora_A.weight': tensor([[-2.9747e-03, 1.7645e-05, -1.3959e-02, ..., 1.6702e-02,\n", - " 2.5911e-02, -1.3416e-02],\n", - " [ 1.5099e-02, 3.0057e-02, -2.4511e-02, ..., -9.5109e-03,\n", - " -1.4320e-02, -3.5647e-02],\n", - " [ 8.3559e-03, -2.1729e-02, 6.0549e-03, ..., 9.1296e-04,\n", - " 7.1130e-03, 6.5738e-03],\n", - " ...,\n", - " [ 2.6930e-02, -1.5346e-02, 8.9250e-03, ..., 2.1033e-02,\n", - " -1.1156e-02, 1.5283e-02],\n", - " [ 3.1569e-02, -1.3637e-02, -4.1950e-02, ..., -1.9484e-02,\n", - " 4.6394e-02, -1.6220e-02],\n", - " [ 3.5849e-03, 3.0520e-02, -2.6064e-02, ..., -2.2740e-02,\n", - " 4.4686e-02, 4.4240e-02]], device='cuda:0'), 'base_model.model.encoder.block.21.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0174, -0.0201, -0.0169, ..., 0.0173, 0.0179, 0.0225],\n", - " [ 0.0099, -0.0035, -0.0075, ..., -0.0073, -0.0029, -0.0030],\n", - " [ 0.0022, -0.0011, -0.0103, ..., 0.0097, 0.0104, 0.0053],\n", - " ...,\n", - " [ 0.0152, -0.0175, 0.0037, ..., 0.0109, -0.0065, 0.0123],\n", - " [-0.0031, -0.0083, 0.0029, ..., 0.0136, -0.0016, -0.0011],\n", - " [-0.0010, 0.0170, -0.0216, ..., -0.0187, 0.0204, 0.0015]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.22.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0314, -0.0045, -0.0207, ..., -0.0543, -0.0126, 0.0270],\n", - " [ 0.0462, 0.0351, -0.0017, ..., 0.0194, 0.0221, 0.0235],\n", - " [ 0.0180, 0.0013, 0.0002, ..., -0.0235, -0.0701, -0.0042],\n", - " ...,\n", - " [-0.0203, -0.0024, 0.0114, ..., -0.0052, -0.0093, -0.0337],\n", - " [-0.0250, 0.0583, 0.0322, ..., 0.0131, 0.0062, -0.0221],\n", - " [-0.0365, -0.0355, 0.0022, ..., -0.0091, -0.0091, -0.0029]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.22.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0084, 0.0145, -0.0058, ..., 0.0047, 0.0110, -0.0105],\n", - " [ 0.0115, 0.0224, 0.0182, ..., -0.0159, -0.0033, -0.0011],\n", - " [-0.0078, 0.0098, -0.0079, ..., 0.0062, 0.0060, 0.0024],\n", - " ...,\n", - " [ 0.0142, -0.0038, 0.0094, ..., -0.0021, -0.0138, 0.0140],\n", - " [ 0.0071, -0.0059, -0.0077, ..., 0.0050, -0.0060, 0.0048],\n", - " [ 0.0117, -0.0122, -0.0040, ..., 0.0117, -0.0074, 0.0111]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.22.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0403, 0.0288, 0.0345, ..., -0.0313, 0.0456, 0.0290],\n", - " [ 0.0169, -0.0029, -0.0118, ..., -0.0086, 0.0247, -0.0073],\n", - " [-0.0182, -0.0154, -0.0023, ..., 0.0213, 0.0123, -0.0090],\n", - " ...,\n", - " [ 0.0398, -0.0134, -0.0142, ..., -0.0308, 0.0373, 0.0003],\n", - " [-0.0289, 0.0193, -0.0009, ..., 0.0243, -0.0158, -0.0300],\n", - " [-0.0136, -0.0490, -0.0062, ..., 0.0037, -0.0265, -0.0094]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.22.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 1.1087e-05, -2.0695e-02, -1.8504e-02, ..., -2.3577e-02,\n", - " -1.6880e-02, 2.1301e-02],\n", - " [-2.5760e-02, -1.2344e-02, -8.9854e-03, ..., 2.8194e-03,\n", - " -1.9121e-02, 9.2568e-03],\n", - " [-2.2456e-03, -1.0488e-02, -7.2482e-03, ..., -1.6379e-02,\n", - " -7.3226e-03, 1.2839e-02],\n", - " ...,\n", - " [-8.2132e-03, -1.3901e-02, -1.9611e-02, ..., -1.4056e-02,\n", - " -1.2747e-02, 1.2911e-02],\n", - " [ 6.6918e-04, -9.2962e-03, -1.1142e-02, ..., -8.8167e-03,\n", - " -1.0819e-02, 1.0372e-02],\n", - " [-6.8010e-03, 1.8507e-02, 2.1169e-02, ..., 1.6902e-02,\n", - " 2.3338e-02, -1.5173e-02]], device='cuda:0'), 'base_model.model.encoder.block.23.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0287, 0.0413, -0.0387, ..., 0.0297, -0.0248, 0.0058],\n", - " [ 0.0030, 0.0372, 0.0231, ..., -0.0069, -0.0422, 0.0422],\n", - " [ 0.0030, 0.0138, -0.0341, ..., 0.0274, -0.0051, -0.0047],\n", - " ...,\n", - " [-0.0029, -0.0164, -0.0410, ..., 0.0089, -0.0333, -0.0027],\n", - " [-0.0257, -0.0437, 0.0370, ..., -0.0213, 0.0096, -0.0510],\n", - " [-0.0032, -0.0417, -0.0039, ..., -0.0076, 0.0019, -0.0326]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.23.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0257, -0.0177, -0.0280, ..., -0.0275, 0.0045, 0.0162],\n", - " [ 0.0056, 0.0036, 0.0022, ..., 0.0057, -0.0219, -0.0013],\n", - " [ 0.0139, 0.0069, 0.0139, ..., 0.0051, 0.0040, -0.0113],\n", - " ...,\n", - " [ 0.0008, -0.0016, -0.0095, ..., 0.0090, 0.0071, 0.0025],\n", - " [-0.0113, -0.0092, -0.0026, ..., -0.0123, -0.0069, 0.0113],\n", - " [ 0.0099, 0.0207, 0.0191, ..., 0.0130, -0.0135, -0.0131]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.23.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0257, -0.0324, 0.0004, ..., -0.0116, 0.0031, 0.0029],\n", - " [-0.0007, -0.0106, -0.0271, ..., -0.0073, 0.0254, 0.0318],\n", - " [ 0.0251, -0.0242, 0.0158, ..., 0.0124, -0.0365, 0.0444],\n", - " ...,\n", - " [ 0.0357, 0.0193, 0.0192, ..., 0.0061, -0.0062, -0.0068],\n", - " [-0.0283, -0.0168, 0.0212, ..., 0.0222, 0.0310, -0.0504],\n", - " [ 0.0458, 0.0061, 0.0074, ..., -0.0495, -0.0247, 0.0176]],\n", - " device='cuda:0'), 'base_model.model.encoder.block.23.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0123, -0.0024, -0.0170, ..., -0.0145, 0.0109, 0.0031],\n", - " [ 0.0129, -0.0127, -0.0100, ..., -0.0079, 0.0134, -0.0099],\n", - " [ 0.0170, -0.0094, -0.0102, ..., -0.0114, 0.0088, -0.0183],\n", - " ...,\n", - " [ 0.0126, -0.0147, -0.0172, ..., -0.0108, 0.0128, -0.0092],\n", - " [ 0.0133, -0.0093, -0.0106, ..., -0.0100, 0.0082, -0.0139],\n", - " [ 0.0047, -0.0057, -0.0064, ..., -0.0078, 0.0074, 0.0181]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.0.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0189, -0.0005, 0.0231, ..., -0.0206, -0.0043, -0.0048],\n", - " [-0.0072, 0.0131, 0.0362, ..., 0.0150, -0.0252, 0.0271],\n", - " [ 0.0177, 0.0022, -0.0043, ..., 0.0060, 0.0091, -0.0020],\n", - " ...,\n", - " [-0.0144, 0.0091, 0.0296, ..., -0.0143, 0.0180, -0.0202],\n", - " [-0.0311, 0.0043, 0.0205, ..., -0.0082, -0.0271, 0.0369],\n", - " [ 0.0101, 0.0102, -0.0180, ..., -0.0206, 0.0186, 0.0433]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.0.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0065, -0.0055, 0.0070, ..., 0.0076, -0.0065, -0.0067],\n", - " [-0.0073, 0.0057, -0.0076, ..., -0.0072, 0.0075, 0.0079],\n", - " [-0.0062, 0.0056, -0.0066, ..., -0.0065, 0.0065, 0.0077],\n", - " ...,\n", - " [ 0.0048, 0.0183, 0.0174, ..., -0.0067, 0.0044, 0.0170],\n", - " [-0.0057, 0.0079, 0.0116, ..., -0.0142, 0.0133, 0.0104],\n", - " [-0.0040, -0.0191, -0.0187, ..., 0.0082, -0.0056, -0.0184]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.0.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0238, 0.0030, 0.0195, ..., -0.0004, -0.0226, -0.0122],\n", - " [-0.0099, -0.0346, -0.0219, ..., 0.0218, 0.0104, -0.0102],\n", - " [-0.0086, -0.0257, 0.0187, ..., 0.0245, 0.0112, -0.0370],\n", - " ...,\n", - " [ 0.0131, -0.0005, -0.0091, ..., -0.0176, -0.0367, -0.0028],\n", - " [ 0.0249, 0.0116, 0.0185, ..., -0.0210, 0.0073, 0.0264],\n", - " [-0.0072, -0.0039, -0.0250, ..., 0.0153, 0.0267, -0.0015]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.0.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0131, 0.0152, 0.0148, ..., 0.0143, -0.0135, -0.0138],\n", - " [-0.0036, 0.0048, 0.0048, ..., 0.0043, -0.0045, -0.0045],\n", - " [-0.0127, 0.0133, 0.0084, ..., 0.0093, -0.0077, -0.0085],\n", - " ...,\n", - " [-0.0257, 0.0265, 0.0275, ..., 0.0272, -0.0283, -0.0266],\n", - " [ 0.0154, -0.0107, -0.0156, ..., -0.0120, 0.0129, 0.0101],\n", - " [-0.0262, 0.0268, 0.0253, ..., 0.0273, -0.0244, -0.0263]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.0.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0221, -0.0252, 0.0064, ..., -0.0018, -0.0022, -0.0222],\n", - " [-0.0064, 0.0094, -0.0091, ..., 0.0109, -0.0069, -0.0229],\n", - " [-0.0291, 0.0128, 0.0075, ..., -0.0042, -0.0206, 0.0037],\n", - " ...,\n", - " [-0.0383, -0.0205, 0.0227, ..., -0.0033, -0.0002, 0.0206],\n", - " [ 0.0220, 0.0185, -0.0156, ..., 0.0298, 0.0298, -0.0049],\n", - " [ 0.0445, -0.0044, -0.0248, ..., 0.0415, -0.0142, 0.0047]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.0.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0167, 0.0058, 0.0135, ..., 0.0126, -0.0094, -0.0168],\n", - " [ 0.0015, 0.0061, 0.0088, ..., 0.0045, -0.0071, -0.0033],\n", - " [-0.0220, 0.0031, -0.0131, ..., -0.0210, 0.0146, 0.0125],\n", - " ...,\n", - " [ 0.0155, -0.0182, 0.0175, ..., 0.0183, -0.0085, -0.0135],\n", - " [ 0.0049, 0.0088, 0.0103, ..., 0.0133, -0.0007, -0.0076],\n", - " [-0.0018, -0.0202, -0.0031, ..., 0.0013, 0.0120, 0.0030]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.0.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0206, -0.0007, 0.0363, ..., -0.0155, 0.0125, 0.0302],\n", - " [ 0.0071, -0.0351, -0.0071, ..., -0.0167, 0.0122, -0.0117],\n", - " [ 0.0421, -0.0014, -0.0129, ..., -0.0072, -0.0114, -0.0469],\n", - " ...,\n", - " [-0.0363, -0.0197, 0.0016, ..., -0.0347, -0.0018, 0.0446],\n", - " [ 0.0193, 0.0251, -0.0119, ..., 0.0415, 0.0172, -0.0267],\n", - " [ 0.0162, 0.0387, -0.0214, ..., -0.0331, 0.0331, -0.0025]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.0.layer.1.EncDecAttention.v.lora_B.weight': tensor([[-0.0207, 0.0257, 0.0249, ..., -0.0258, 0.0105, -0.0223],\n", - " [-0.0098, 0.0059, 0.0131, ..., -0.0124, -0.0172, -0.0154],\n", - " [-0.0001, -0.0220, 0.0026, ..., 0.0013, -0.0207, 0.0071],\n", - " ...,\n", - " [-0.0171, 0.0202, 0.0187, ..., -0.0191, -0.0165, -0.0220],\n", - " [ 0.0091, -0.0049, -0.0022, ..., 0.0102, 0.0082, 0.0143],\n", - " [ 0.0122, -0.0073, -0.0114, ..., 0.0135, -0.0045, 0.0117]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.1.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0062, 0.0340, -0.0126, ..., -0.0317, -0.0148, 0.0114],\n", - " [ 0.0016, 0.0201, -0.0259, ..., -0.0081, -0.0180, 0.0036],\n", - " [-0.0196, -0.0027, 0.0157, ..., -0.0040, -0.0164, -0.0303],\n", - " ...,\n", - " [-0.0307, -0.0401, 0.0096, ..., -0.0108, 0.0102, -0.0025],\n", - " [ 0.0146, 0.0296, 0.0074, ..., 0.0368, 0.0319, 0.0163],\n", - " [ 0.0212, 0.0060, -0.0247, ..., 0.0213, 0.0096, 0.0179]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.1.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0083, 0.0084, -0.0099, ..., 0.0071, -0.0083, -0.0079],\n", - " [-0.0076, -0.0083, 0.0089, ..., -0.0073, 0.0083, 0.0083],\n", - " [-0.0078, -0.0082, 0.0091, ..., -0.0071, 0.0082, 0.0078],\n", - " ...,\n", - " [ 0.0186, -0.0079, -0.0116, ..., -0.0011, 0.0153, 0.0145],\n", - " [ 0.0055, -0.0129, 0.0063, ..., -0.0055, -0.0006, 0.0057],\n", - " [-0.0068, 0.0094, 0.0049, ..., -0.0214, -0.0105, -0.0143]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.1.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0069, 0.0235, -0.0279, ..., 0.0227, -0.0086, 0.0170],\n", - " [-0.0380, -0.0010, 0.0127, ..., -0.0040, -0.0202, -0.0155],\n", - " [ 0.0048, -0.0332, 0.0042, ..., 0.0085, -0.0161, -0.0132],\n", - " ...,\n", - " [ 0.0162, -0.0120, -0.0343, ..., 0.0348, -0.0002, 0.0094],\n", - " [ 0.0057, -0.0085, -0.0151, ..., -0.0225, -0.0477, -0.0247],\n", - " [ 0.0267, -0.0135, -0.0142, ..., 0.0340, -0.0055, -0.0230]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.1.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0185, -0.0192, 0.0192, ..., 0.0191, -0.0146, 0.0189],\n", - " [-0.0069, -0.0093, 0.0093, ..., 0.0110, -0.0066, 0.0097],\n", - " [-0.0158, -0.0179, 0.0180, ..., 0.0171, -0.0151, 0.0170],\n", - " ...,\n", - " [ 0.0105, 0.0053, -0.0043, ..., -0.0051, 0.0076, -0.0062],\n", - " [-0.0061, -0.0023, 0.0024, ..., 0.0026, -0.0050, 0.0019],\n", - " [-0.0016, -0.0042, 0.0057, ..., 0.0055, 0.0083, 0.0062]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.1.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0090, 0.0097, 0.0331, ..., 0.0069, -0.0390, -0.0076],\n", - " [-0.0125, 0.0190, -0.0041, ..., 0.0240, -0.0379, 0.0208],\n", - " [ 0.0228, -0.0280, 0.0044, ..., 0.0122, 0.0094, 0.0115],\n", - " ...,\n", - " [-0.0252, 0.0296, 0.0183, ..., -0.0186, 0.0372, 0.0183],\n", - " [-0.0111, -0.0163, 0.0142, ..., 0.0146, -0.0015, -0.0034],\n", - " [ 0.0198, 0.0215, -0.0137, ..., -0.0140, 0.0162, 0.0132]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.1.layer.1.EncDecAttention.q.lora_B.weight': tensor([[-0.0008, 0.0053, 0.0051, ..., 0.0054, -0.0029, 0.0046],\n", - " [ 0.0004, 0.0072, 0.0072, ..., 0.0089, 0.0025, 0.0062],\n", - " [ 0.0051, -0.0091, -0.0187, ..., -0.0199, 0.0004, -0.0147],\n", - " ...,\n", - " [ 0.0021, 0.0219, 0.0079, ..., 0.0080, 0.0092, 0.0068],\n", - " [-0.0025, -0.0209, -0.0022, ..., -0.0014, -0.0061, -0.0011],\n", - " [ 0.0134, 0.0163, -0.0054, ..., -0.0052, 0.0140, -0.0088]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.1.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0112, 0.0366, -0.0094, ..., -0.0264, 0.0193, 0.0159],\n", - " [-0.0172, -0.0171, -0.0273, ..., 0.0062, -0.0069, -0.0036],\n", - " [ 0.0194, 0.0024, -0.0140, ..., -0.0079, 0.0263, -0.0028],\n", - " ...,\n", - " [ 0.0128, -0.0109, -0.0209, ..., -0.0019, -0.0316, -0.0220],\n", - " [ 0.0199, 0.0126, 0.0240, ..., -0.0239, 0.0473, 0.0146],\n", - " [ 0.0186, 0.0341, -0.0052, ..., -0.0248, 0.0462, 0.0117]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.1.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 2.3198e-02, -2.1747e-02, 2.1701e-02, ..., -2.1566e-02,\n", - " 2.4150e-02, 2.1774e-02],\n", - " [-1.6284e-03, 7.0164e-04, -1.1504e-03, ..., 4.7365e-03,\n", - " -9.8117e-05, -6.4463e-03],\n", - " [ 1.5113e-02, -1.2611e-02, 1.4666e-02, ..., -9.9986e-03,\n", - " 1.1651e-02, 2.0736e-03],\n", - " ...,\n", - " [ 6.9344e-03, -3.2302e-03, 5.0286e-03, ..., -7.6372e-03,\n", - " 3.3706e-04, 3.2491e-04],\n", - " [ 1.6321e-02, -6.3958e-03, 5.0811e-03, ..., -4.7124e-03,\n", - " 3.7874e-03, 3.6046e-03],\n", - " [-9.5407e-04, -9.5503e-03, 1.0742e-02, ..., -4.9108e-03,\n", - " 6.3167e-03, 5.7234e-03]], device='cuda:0'), 'base_model.model.decoder.block.2.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0043, 0.0264, -0.0115, ..., 0.0311, 0.0349, 0.0248],\n", - " [-0.0181, -0.0033, 0.0367, ..., 0.0160, -0.0377, -0.0214],\n", - " [ 0.0447, -0.0075, -0.0205, ..., 0.0276, 0.0195, -0.0050],\n", - " ...,\n", - " [-0.0450, -0.0030, 0.0271, ..., -0.0239, 0.0129, 0.0078],\n", - " [-0.0393, -0.0002, 0.0071, ..., -0.0083, 0.0064, -0.0282],\n", - " [ 0.0125, -0.0222, 0.0274, ..., -0.0385, -0.0269, -0.0489]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.2.layer.0.SelfAttention.q.lora_B.weight': tensor([[-3.1170e-03, 1.0720e-02, 2.1632e-03, ..., -8.6142e-04,\n", - " -1.2131e-03, 3.1180e-03],\n", - " [ 6.1501e-03, -1.2021e-02, 3.3474e-07, ..., -8.4009e-03,\n", - " -3.4500e-03, -8.8669e-03],\n", - " [-7.0843e-03, 5.7099e-04, -1.1848e-02, ..., 2.5516e-03,\n", - " 5.1409e-03, 1.3616e-03],\n", - " ...,\n", - " [ 2.2997e-03, 5.7992e-03, -2.0466e-02, ..., 1.0433e-02,\n", - " 1.6501e-02, -2.6774e-03],\n", - " [-6.2623e-03, 4.4403e-04, -1.5141e-02, ..., 6.5718e-03,\n", - " 1.8738e-02, 4.3951e-03],\n", - " [-4.7032e-03, 5.6032e-03, -1.0028e-02, ..., 9.1350e-03,\n", - " 8.2068e-03, 4.3547e-03]], device='cuda:0'), 'base_model.model.decoder.block.2.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0162, -0.0015, 0.0074, ..., -0.0003, 0.0158, -0.0235],\n", - " [-0.0018, -0.0206, 0.0025, ..., 0.0132, -0.0139, 0.0327],\n", - " [ 0.0019, -0.0290, -0.0073, ..., -0.0219, 0.0109, 0.0003],\n", - " ...,\n", - " [ 0.0291, -0.0131, -0.0077, ..., -0.0039, -0.0046, -0.0121],\n", - " [ 0.0109, -0.0214, 0.0038, ..., 0.0181, -0.0220, -0.0040],\n", - " [-0.0069, -0.0031, 0.0367, ..., -0.0086, 0.0165, 0.0042]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.2.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 2.4285e-02, 1.9491e-02, 2.0244e-02, ..., 2.0564e-02,\n", - " -2.1249e-02, 2.2550e-02],\n", - " [ 1.6146e-02, 2.1073e-02, 2.0157e-02, ..., 2.2195e-02,\n", - " -1.9886e-02, 1.3432e-02],\n", - " [-2.4051e-03, -3.1521e-03, -2.6053e-03, ..., -8.9358e-03,\n", - " 2.8409e-03, -6.3286e-03],\n", - " ...,\n", - " [ 8.3711e-05, -5.0593e-03, 6.4664e-04, ..., -3.6170e-03,\n", - " -2.2688e-05, -3.6618e-03],\n", - " [ 2.6448e-02, 2.5704e-02, 2.2131e-02, ..., 2.7836e-02,\n", - " -2.6099e-02, 1.0297e-02],\n", - " [-2.4451e-02, -2.4363e-02, -2.2445e-02, ..., -2.3449e-02,\n", - " 2.3975e-02, -2.5908e-02]], device='cuda:0'), 'base_model.model.decoder.block.2.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-3.2235e-02, -2.4453e-02, -1.8897e-03, ..., -6.8799e-03,\n", - " -4.4862e-02, -1.3606e-02],\n", - " [ 2.0254e-02, -1.5936e-02, 4.4509e-03, ..., 2.1429e-04,\n", - " -1.0414e-02, 9.5917e-03],\n", - " [-3.7509e-02, -7.7726e-03, 8.3584e-03, ..., 8.5235e-05,\n", - " -4.5792e-02, 1.7381e-02],\n", - " ...,\n", - " [ 1.3659e-02, -3.2639e-02, 1.3874e-02, ..., -9.9682e-03,\n", - " 7.1122e-03, -1.3168e-02],\n", - " [ 2.7758e-02, 1.7779e-02, 2.0833e-02, ..., 9.7087e-03,\n", - " 4.2300e-02, 1.4540e-02],\n", - " [-4.2461e-03, 3.1331e-03, 2.5784e-02, ..., -3.8201e-02,\n", - " 6.7687e-03, -1.6261e-02]], device='cuda:0'), 'base_model.model.decoder.block.2.layer.1.EncDecAttention.q.lora_B.weight': tensor([[-0.0192, -0.0265, -0.0246, ..., 0.0291, 0.0167, -0.0226],\n", - " [ 0.0060, 0.0069, 0.0018, ..., -0.0137, -0.0044, 0.0027],\n", - " [ 0.0109, 0.0068, 0.0197, ..., -0.0124, -0.0072, 0.0091],\n", - " ...,\n", - " [ 0.0187, -0.0130, 0.0149, ..., -0.0143, -0.0120, 0.0074],\n", - " [-0.0028, 0.0222, 0.0006, ..., -0.0003, 0.0002, 0.0081],\n", - " [-0.0265, -0.0019, -0.0296, ..., 0.0252, 0.0172, -0.0095]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.2.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0110, -0.0249, -0.0031, ..., -0.0298, 0.0260, 0.0516],\n", - " [-0.0553, 0.0024, 0.0013, ..., -0.0013, 0.0018, 0.0188],\n", - " [-0.0096, -0.0039, -0.0053, ..., -0.0048, 0.0163, 0.0174],\n", - " ...,\n", - " [-0.0267, -0.0287, 0.0545, ..., 0.0190, 0.0132, 0.0087],\n", - " [-0.0091, 0.0195, 0.0129, ..., -0.0401, 0.0185, 0.0505],\n", - " [ 0.0057, -0.0250, 0.0268, ..., -0.0211, 0.0299, 0.0019]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.2.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 1.3512e-02, 1.0094e-02, 1.0749e-02, ..., 1.3937e-02,\n", - " 1.4577e-02, 1.3163e-02],\n", - " [-1.6808e-02, -1.2401e-02, -2.2549e-02, ..., -1.5579e-02,\n", - " -1.5503e-02, -1.6389e-02],\n", - " [-6.7631e-03, -4.8791e-03, -9.9732e-03, ..., -8.7470e-03,\n", - " -7.3061e-03, -9.7309e-03],\n", - " ...,\n", - " [ 7.1581e-05, -2.2548e-03, -8.9037e-03, ..., -6.1649e-03,\n", - " -4.6464e-03, -6.3443e-03],\n", - " [-2.1520e-02, -1.6354e-02, -2.1507e-02, ..., -1.8473e-02,\n", - " -2.1734e-02, -2.0071e-02],\n", - " [-2.2858e-04, -1.2750e-03, 5.7743e-04, ..., 1.1762e-03,\n", - " 2.3638e-03, 2.9686e-03]], device='cuda:0'), 'base_model.model.decoder.block.3.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0072, -0.0215, 0.0113, ..., -0.0083, -0.0112, -0.0261],\n", - " [ 0.0192, 0.0184, 0.0049, ..., -0.0295, 0.0275, -0.0116],\n", - " [ 0.0280, 0.0047, -0.0025, ..., 0.0011, -0.0008, 0.0254],\n", - " ...,\n", - " [-0.0189, 0.0099, -0.0184, ..., -0.0014, 0.0009, 0.0082],\n", - " [-0.0026, -0.0022, 0.0208, ..., -0.0045, 0.0279, -0.0247],\n", - " [ 0.0204, -0.0134, -0.0076, ..., 0.0101, -0.0260, 0.0311]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.3.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0130, 0.0056, 0.0038, ..., -0.0028, -0.0109, 0.0039],\n", - " [ 0.0065, -0.0119, 0.0148, ..., -0.0166, 0.0059, 0.0121],\n", - " [-0.0192, 0.0150, 0.0005, ..., 0.0006, -0.0156, 0.0008],\n", - " ...,\n", - " [-0.0164, -0.0180, 0.0115, ..., -0.0031, -0.0211, 0.0231],\n", - " [ 0.0160, 0.0132, -0.0068, ..., 0.0005, 0.0191, -0.0242],\n", - " [ 0.0048, -0.0069, 0.0201, ..., -0.0211, -0.0003, 0.0144]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.3.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0154, -0.0190, -0.0139, ..., -0.0068, 0.0060, -0.0097],\n", - " [-0.0084, 0.0118, -0.0295, ..., -0.0322, -0.0392, -0.0278],\n", - " [ 0.0255, -0.0257, 0.0247, ..., 0.0118, -0.0110, 0.0428],\n", - " ...,\n", - " [ 0.0037, 0.0066, 0.0163, ..., 0.0142, -0.0087, -0.0071],\n", - " [ 0.0321, 0.0076, 0.0286, ..., -0.0035, 0.0288, 0.0387],\n", - " [-0.0373, -0.0299, -0.0162, ..., -0.0288, -0.0450, -0.0064]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.3.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0055, 0.0059, -0.0068, ..., 0.0066, -0.0059, 0.0058],\n", - " [ 0.0017, 0.0039, -0.0019, ..., 0.0005, 0.0024, -0.0008],\n", - " [-0.0046, -0.0104, 0.0097, ..., -0.0098, 0.0082, -0.0097],\n", - " ...,\n", - " [-0.0010, 0.0006, 0.0019, ..., 0.0017, -0.0012, -0.0005],\n", - " [ 0.0027, 0.0070, -0.0015, ..., 0.0019, -0.0003, 0.0019],\n", - " [-0.0111, -0.0121, 0.0130, ..., -0.0129, 0.0124, -0.0135]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.3.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0305, 0.0111, 0.0111, ..., 0.0119, -0.0292, -0.0201],\n", - " [ 0.0215, 0.0097, -0.0449, ..., -0.0125, 0.0371, 0.0259],\n", - " [-0.0006, -0.0228, 0.0173, ..., -0.0298, -0.0348, -0.0166],\n", - " ...,\n", - " [-0.0496, -0.0060, 0.0336, ..., -0.0376, -0.0366, 0.0158],\n", - " [ 0.0150, -0.0011, -0.0006, ..., 0.0357, 0.0258, 0.0381],\n", - " [-0.0085, -0.0071, 0.0198, ..., 0.0362, -0.0071, 0.0058]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.3.layer.1.EncDecAttention.q.lora_B.weight': tensor([[-0.0022, -0.0042, 0.0066, ..., 0.0047, -0.0036, -0.0020],\n", - " [ 0.0189, -0.0371, 0.0382, ..., 0.0305, -0.0319, -0.0206],\n", - " [-0.0052, -0.0098, 0.0078, ..., 0.0078, -0.0058, -0.0184],\n", - " ...,\n", - " [ 0.0121, 0.0118, -0.0107, ..., -0.0072, 0.0152, -0.0095],\n", - " [-0.0156, 0.0325, -0.0318, ..., -0.0103, 0.0305, 0.0194],\n", - " [ 0.0130, 0.0154, -0.0188, ..., -0.0170, 0.0142, 0.0074]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.3.layer.1.EncDecAttention.v.lora_A.weight': tensor([[ 0.0188, -0.0262, 0.0216, ..., 0.0069, -0.0215, -0.0048],\n", - " [ 0.0322, -0.0194, 0.0094, ..., 0.0157, -0.0021, 0.0539],\n", - " [-0.0159, 0.0084, 0.0091, ..., -0.0113, 0.0037, 0.0314],\n", - " ...,\n", - " [ 0.0004, -0.0198, 0.0015, ..., -0.0551, 0.0388, 0.0306],\n", - " [ 0.0350, -0.0306, -0.0190, ..., 0.0378, -0.0301, -0.0262],\n", - " [ 0.0203, 0.0142, -0.0191, ..., -0.0139, 0.0464, 0.0181]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.3.layer.1.EncDecAttention.v.lora_B.weight': tensor([[-0.0019, -0.0041, 0.0024, ..., -0.0003, -0.0006, 0.0029],\n", - " [-0.0188, 0.0186, 0.0175, ..., 0.0169, -0.0168, 0.0166],\n", - " [ 0.0126, -0.0167, -0.0079, ..., -0.0090, 0.0082, -0.0114],\n", - " ...,\n", - " [-0.0065, 0.0139, 0.0076, ..., 0.0082, -0.0083, 0.0121],\n", - " [-0.0111, 0.0149, 0.0127, ..., 0.0131, -0.0134, 0.0118],\n", - " [ 0.0017, 0.0036, -0.0023, ..., -0.0028, 0.0031, -0.0028]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.4.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0342, -0.0317, -0.0014, ..., -0.0271, 0.0120, 0.0191],\n", - " [-0.0287, -0.0342, 0.0428, ..., 0.0099, -0.0305, 0.0277],\n", - " [-0.0308, -0.0273, -0.0005, ..., -0.0049, -0.0236, -0.0056],\n", - " ...,\n", - " [-0.0138, -0.0167, -0.0405, ..., 0.0315, 0.0343, 0.0241],\n", - " [-0.0294, -0.0314, 0.0403, ..., 0.0076, -0.0130, -0.0139],\n", - " [-0.0245, -0.0102, 0.0352, ..., -0.0369, -0.0388, 0.0226]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.4.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0039, 0.0033, 0.0120, ..., 0.0007, -0.0020, 0.0129],\n", - " [ 0.0154, 0.0150, 0.0096, ..., -0.0125, 0.0154, 0.0157],\n", - " [ 0.0114, 0.0126, 0.0129, ..., -0.0122, 0.0115, 0.0021],\n", - " ...,\n", - " [-0.0076, -0.0078, -0.0041, ..., -0.0005, -0.0058, -0.0095],\n", - " [ 0.0091, 0.0096, 0.0089, ..., -0.0087, 0.0105, 0.0073],\n", - " [-0.0055, -0.0083, -0.0074, ..., 0.0106, -0.0107, -0.0063]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.4.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0116, 0.0151, -0.0338, ..., 0.0024, 0.0069, 0.0019],\n", - " [-0.0075, 0.0279, -0.0158, ..., -0.0128, 0.0325, 0.0015],\n", - " [ 0.0325, -0.0339, 0.0060, ..., -0.0077, 0.0005, 0.0309],\n", - " ...,\n", - " [ 0.0232, 0.0226, -0.0187, ..., 0.0152, -0.0232, 0.0162],\n", - " [ 0.0283, 0.0220, 0.0349, ..., 0.0378, 0.0139, 0.0358],\n", - " [ 0.0147, 0.0302, -0.0180, ..., 0.0072, -0.0060, -0.0410]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.4.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0115, 0.0098, 0.0073, ..., 0.0100, 0.0085, -0.0100],\n", - " [ 0.0131, -0.0123, -0.0133, ..., -0.0141, -0.0025, 0.0153],\n", - " [-0.0161, 0.0144, 0.0136, ..., 0.0137, 0.0108, -0.0134],\n", - " ...,\n", - " [-0.0047, 0.0031, 0.0027, ..., 0.0071, 0.0149, -0.0057],\n", - " [ 0.0156, -0.0158, -0.0076, ..., -0.0147, -0.0157, 0.0160],\n", - " [-0.0158, 0.0169, 0.0219, ..., 0.0165, 0.0194, -0.0162]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.4.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0263, 0.0215, 0.0172, ..., -0.0139, -0.0270, -0.0057],\n", - " [ 0.0239, -0.0136, 0.0008, ..., 0.0056, 0.0047, -0.0103],\n", - " [ 0.0140, 0.0015, -0.0131, ..., 0.0060, 0.0066, 0.0215],\n", - " ...,\n", - " [ 0.0302, -0.0049, -0.0103, ..., 0.0099, -0.0047, -0.0163],\n", - " [ 0.0014, 0.0275, 0.0259, ..., 0.0201, -0.0150, 0.0098],\n", - " [ 0.0081, 0.0232, -0.0162, ..., -0.0027, -0.0194, 0.0108]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.4.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0242, 0.0186, -0.0200, ..., 0.0223, -0.0155, 0.0179],\n", - " [-0.0203, -0.0142, 0.0140, ..., -0.0150, 0.0092, -0.0160],\n", - " [-0.0007, 0.0006, -0.0005, ..., 0.0006, -0.0002, -0.0026],\n", - " ...,\n", - " [ 0.0092, 0.0041, -0.0047, ..., 0.0082, -0.0055, 0.0086],\n", - " [ 0.0151, 0.0137, -0.0022, ..., 0.0107, -0.0017, 0.0091],\n", - " [-0.0118, -0.0104, 0.0048, ..., -0.0077, -0.0013, -0.0059]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.4.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0023, -0.0160, 0.0328, ..., 0.0381, 0.0126, -0.0562],\n", - " [-0.0314, 0.0148, 0.0142, ..., 0.0375, -0.0140, -0.0295],\n", - " [-0.0111, 0.0272, 0.0154, ..., 0.0156, 0.0067, -0.0610],\n", - " ...,\n", - " [-0.0016, -0.0208, 0.0008, ..., 0.0163, 0.0187, 0.0313],\n", - " [ 0.0035, -0.0200, 0.0226, ..., -0.0011, -0.0050, 0.0417],\n", - " [ 0.0164, 0.0416, -0.0165, ..., -0.0197, 0.0283, -0.0016]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.4.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 0.0080, 0.0172, 0.0179, ..., 0.0031, -0.0199, -0.0094],\n", - " [ 0.0060, 0.0121, 0.0121, ..., -0.0026, -0.0113, -0.0111],\n", - " [-0.0091, -0.0273, -0.0263, ..., 0.0116, 0.0272, 0.0177],\n", - " ...,\n", - " [-0.0019, -0.0016, 0.0025, ..., 0.0041, 0.0043, 0.0018],\n", - " [-0.0042, -0.0017, -0.0011, ..., -0.0032, -0.0010, 0.0023],\n", - " [ 0.0004, 0.0092, 0.0103, ..., -0.0058, -0.0173, -0.0072]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.5.layer.0.SelfAttention.q.lora_A.weight': tensor([[-2.6730e-02, 2.0864e-02, 2.3388e-02, ..., -1.5293e-02,\n", - " -1.6101e-03, -4.0400e-02],\n", - " [-3.2931e-02, -1.0590e-03, 1.2445e-02, ..., -5.5235e-03,\n", - " 3.3820e-02, -2.0991e-02],\n", - " [ 1.2719e-02, 2.5506e-03, -3.6293e-02, ..., 1.6096e-02,\n", - " 1.3012e-02, 1.6014e-02],\n", - " ...,\n", - " [-1.3189e-02, -2.4657e-02, -8.4241e-03, ..., -5.5609e-03,\n", - " -4.4067e-03, -3.3457e-02],\n", - " [ 2.7041e-02, 2.7457e-02, -2.4774e-03, ..., 2.2314e-03,\n", - " 1.1362e-02, -9.3929e-03],\n", - " [ 1.5837e-02, -9.2169e-03, -7.3251e-04, ..., -2.6933e-05,\n", - " 1.2411e-02, -4.1066e-03]], device='cuda:0'), 'base_model.model.decoder.block.5.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0014, 0.0082, 0.0049, ..., -0.0001, -0.0062, 0.0002],\n", - " [-0.0049, -0.0086, -0.0027, ..., -0.0024, 0.0020, 0.0039],\n", - " [-0.0048, -0.0091, -0.0022, ..., -0.0030, 0.0021, 0.0041],\n", - " ...,\n", - " [-0.0004, 0.0162, 0.0006, ..., 0.0005, -0.0079, 0.0013],\n", - " [-0.0036, -0.0084, 0.0063, ..., -0.0062, 0.0043, 0.0085],\n", - " [-0.0044, 0.0027, 0.0058, ..., -0.0065, -0.0014, 0.0068]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.5.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0186, -0.0075, 0.0282, ..., -0.0128, -0.0348, -0.0162],\n", - " [ 0.0120, 0.0150, -0.0364, ..., -0.0095, 0.0025, -0.0103],\n", - " [ 0.0250, 0.0160, -0.0067, ..., -0.0135, 0.0271, 0.0212],\n", - " ...,\n", - " [ 0.0176, -0.0096, 0.0075, ..., 0.0166, -0.0108, -0.0156],\n", - " [ 0.0347, 0.0249, -0.0111, ..., -0.0084, 0.0292, 0.0088],\n", - " [-0.0105, 0.0333, 0.0109, ..., -0.0365, 0.0088, -0.0124]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.5.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0106, 0.0103, 0.0088, ..., 0.0104, 0.0139, -0.0114],\n", - " [ 0.0053, -0.0025, -0.0100, ..., -0.0028, -0.0017, 0.0061],\n", - " [ 0.0084, -0.0051, -0.0125, ..., -0.0073, -0.0063, 0.0097],\n", - " ...,\n", - " [ 0.0039, 0.0021, -0.0035, ..., -0.0035, -0.0005, 0.0048],\n", - " [-0.0107, 0.0107, 0.0084, ..., 0.0114, 0.0113, -0.0080],\n", - " [ 0.0057, -0.0105, -0.0100, ..., -0.0036, -0.0060, 0.0014]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.5.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0253, 0.0348, 0.0144, ..., 0.0535, -0.0086, -0.0021],\n", - " [ 0.0331, -0.0260, 0.0130, ..., -0.0028, -0.0292, 0.0152],\n", - " [ 0.0248, -0.0087, -0.0264, ..., -0.0248, -0.0025, 0.0110],\n", - " ...,\n", - " [ 0.0223, -0.0317, 0.0493, ..., -0.0156, -0.0037, -0.0297],\n", - " [ 0.0025, 0.0178, -0.0461, ..., -0.0236, 0.0002, 0.0295],\n", - " [ 0.0259, -0.0477, 0.0526, ..., -0.0363, -0.0160, -0.0426]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.5.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0095, -0.0096, -0.0092, ..., -0.0071, 0.0069, -0.0087],\n", - " [ 0.0015, -0.0024, -0.0026, ..., -0.0026, 0.0081, -0.0055],\n", - " [-0.0016, 0.0003, 0.0006, ..., 0.0035, -0.0067, -0.0027],\n", - " ...,\n", - " [ 0.0003, 0.0031, -0.0029, ..., 0.0003, 0.0018, -0.0005],\n", - " [-0.0036, 0.0015, 0.0029, ..., 0.0102, -0.0047, 0.0026],\n", - " [-0.0088, 0.0082, 0.0067, ..., -0.0148, 0.0157, -0.0170]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.5.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0108, -0.0207, -0.0201, ..., -0.0062, -0.0595, -0.0536],\n", - " [ 0.0466, 0.0231, 0.0192, ..., 0.0111, 0.0117, 0.0167],\n", - " [ 0.0248, -0.0007, -0.0118, ..., -0.0208, 0.0139, 0.0163],\n", - " ...,\n", - " [ 0.0050, -0.0228, 0.0266, ..., 0.0138, 0.0375, 0.0285],\n", - " [ 0.0072, 0.0144, 0.0285, ..., 0.0107, 0.0474, 0.0207],\n", - " [ 0.0098, 0.0003, -0.0225, ..., -0.0130, 0.0355, 0.0492]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.5.layer.1.EncDecAttention.v.lora_B.weight': tensor([[-2.1625e-03, 1.6814e-04, 8.1146e-05, ..., 1.4400e-03,\n", - " -5.4853e-03, -1.1964e-03],\n", - " [ 6.4509e-03, -6.9242e-03, -1.6537e-02, ..., -6.6245e-03,\n", - " -2.1439e-02, -1.5015e-02],\n", - " [-8.9251e-04, 7.1215e-04, -2.7612e-04, ..., 2.6042e-04,\n", - " 1.3185e-02, 9.8170e-03],\n", - " ...,\n", - " [-3.6989e-03, 1.8064e-03, 8.6825e-03, ..., 2.5273e-03,\n", - " 4.6486e-03, 5.4987e-03],\n", - " [-1.1506e-02, 8.4292e-03, 5.8922e-03, ..., 1.1264e-02,\n", - " 4.8699e-03, 7.4301e-03],\n", - " [ 1.0491e-02, -9.8668e-03, -1.0596e-02, ..., -9.7787e-03,\n", - " -5.6693e-03, -9.9868e-03]], device='cuda:0'), 'base_model.model.decoder.block.6.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0067, -0.0038, 0.0387, ..., 0.0008, -0.0110, 0.0104],\n", - " [ 0.0157, -0.0423, 0.0041, ..., -0.0276, 0.0129, 0.0129],\n", - " [-0.0351, 0.0106, -0.0120, ..., 0.0130, -0.0018, -0.0170],\n", - " ...,\n", - " [ 0.0046, 0.0349, 0.0031, ..., 0.0075, 0.0278, 0.0109],\n", - " [ 0.0355, 0.0083, 0.0434, ..., -0.0034, -0.0004, -0.0221],\n", - " [-0.0067, 0.0074, 0.0048, ..., 0.0083, -0.0431, -0.0202]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.6.layer.0.SelfAttention.q.lora_B.weight': tensor([[-1.4334e-02, -2.0122e-02, 2.5548e-02, ..., 1.8705e-02,\n", - " 4.1224e-04, -9.1952e-03],\n", - " [-3.4477e-03, -6.8384e-04, -4.9736e-05, ..., 2.7096e-03,\n", - " 9.1590e-03, 1.4286e-02],\n", - " [-1.1760e-02, -1.6113e-02, 1.9543e-02, ..., 1.4789e-02,\n", - " 3.0643e-03, -6.1182e-03],\n", - " ...,\n", - " [ 1.0866e-02, 1.0069e-02, -8.9908e-03, ..., -9.9777e-03,\n", - " 1.0297e-02, 7.7944e-03],\n", - " [-1.1795e-02, -1.1093e-02, 1.2621e-02, ..., 1.2965e-02,\n", - " -9.7311e-03, -1.1893e-02],\n", - " [ 1.2635e-02, 1.1360e-02, -1.1950e-02, ..., -1.2414e-02,\n", - " 1.0531e-02, 1.4034e-02]], device='cuda:0'), 'base_model.model.decoder.block.6.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0033, -0.0322, 0.0411, ..., -0.0003, -0.0140, 0.0132],\n", - " [ 0.0124, -0.0387, 0.0131, ..., -0.0345, 0.0151, 0.0003],\n", - " [-0.0095, 0.0191, -0.0311, ..., -0.0108, -0.0286, -0.0093],\n", - " ...,\n", - " [-0.0186, 0.0239, -0.0576, ..., -0.0223, -0.0066, -0.0108],\n", - " [ 0.0070, -0.0496, -0.0148, ..., -0.0102, -0.0159, 0.0199],\n", - " [ 0.0071, 0.0019, -0.0213, ..., 0.0124, -0.0183, 0.0216]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.6.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 2.4060e-02, 2.1683e-02, -2.5614e-02, ..., -2.4639e-02,\n", - " 2.5046e-02, 2.4352e-02],\n", - " [ 1.3227e-02, 9.1044e-03, -1.3184e-02, ..., -1.1382e-02,\n", - " 1.2999e-02, 1.3374e-02],\n", - " [-1.4519e-02, -1.2697e-02, 1.2229e-02, ..., 1.1532e-02,\n", - " -1.4862e-02, -1.5476e-02],\n", - " ...,\n", - " [-1.6640e-02, -1.5588e-02, 1.2420e-02, ..., 2.0494e-02,\n", - " -1.3000e-02, -1.1540e-02],\n", - " [ 1.6104e-02, 1.6665e-02, -8.1493e-03, ..., -2.1218e-02,\n", - " 8.9385e-03, 8.0249e-03],\n", - " [ 6.4023e-05, 1.2668e-03, -8.7618e-03, ..., 3.2057e-05,\n", - " 8.1570e-03, 9.9918e-03]], device='cuda:0'), 'base_model.model.decoder.block.6.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0133, 0.0064, 0.0368, ..., 0.0051, -0.0354, -0.0184],\n", - " [-0.0097, 0.0049, -0.0109, ..., -0.0311, -0.0086, -0.0254],\n", - " [-0.0274, 0.0319, -0.0215, ..., 0.0142, 0.0177, 0.0154],\n", - " ...,\n", - " [-0.0239, -0.0018, 0.0185, ..., -0.0107, -0.0099, 0.0124],\n", - " [-0.0030, -0.0054, -0.0009, ..., -0.0291, 0.0086, 0.0105],\n", - " [-0.0358, -0.0142, -0.0217, ..., -0.0015, 0.0169, 0.0144]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.6.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0037, 0.0140, 0.0248, ..., 0.0287, -0.0060, 0.0007],\n", - " [-0.0074, 0.0020, -0.0056, ..., 0.0008, 0.0032, 0.0039],\n", - " [-0.0019, -0.0019, 0.0006, ..., 0.0045, -0.0034, 0.0044],\n", - " ...,\n", - " [ 0.0046, -0.0029, 0.0052, ..., 0.0034, 0.0060, -0.0008],\n", - " [ 0.0097, 0.0037, 0.0133, ..., 0.0158, -0.0098, -0.0095],\n", - " [ 0.0096, -0.0046, 0.0003, ..., -0.0082, 0.0195, -0.0122]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.6.layer.1.EncDecAttention.v.lora_A.weight': tensor([[ 0.0266, -0.0150, -0.0428, ..., 0.0195, 0.0117, 0.0346],\n", - " [ 0.0236, -0.0423, 0.0263, ..., -0.0011, -0.0402, -0.0415],\n", - " [-0.0090, 0.0466, -0.0165, ..., -0.0530, -0.0045, 0.0174],\n", - " ...,\n", - " [-0.0144, -0.0242, 0.0267, ..., 0.0429, -0.0238, -0.0078],\n", - " [ 0.0131, -0.0153, 0.0209, ..., 0.0367, -0.0131, -0.0385],\n", - " [ 0.0002, 0.0393, -0.0139, ..., -0.0045, 0.0177, 0.0329]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.6.layer.1.EncDecAttention.v.lora_B.weight': tensor([[-0.0009, -0.0140, 0.0066, ..., -0.0093, 0.0028, 0.0081],\n", - " [-0.0098, 0.0039, -0.0037, ..., 0.0026, 0.0042, -0.0022],\n", - " [ 0.0084, 0.0001, 0.0004, ..., 0.0047, -0.0066, 0.0003],\n", - " ...,\n", - " [ 0.0044, 0.0213, -0.0220, ..., 0.0198, 0.0182, -0.0189],\n", - " [-0.0040, 0.0051, -0.0058, ..., 0.0041, 0.0030, -0.0046],\n", - " [-0.0051, 0.0010, 0.0045, ..., -0.0046, -0.0037, 0.0040]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.7.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0331, -0.0111, -0.0323, ..., 0.0251, 0.0505, -0.0046],\n", - " [-0.0215, 0.0399, 0.0451, ..., -0.0216, -0.0509, -0.0254],\n", - " [-0.0085, 0.0070, 0.0028, ..., -0.0233, 0.0098, -0.0006],\n", - " ...,\n", - " [-0.0136, -0.0158, 0.0197, ..., -0.0471, 0.0237, -0.0038],\n", - " [ 0.0197, -0.0128, -0.0333, ..., 0.0287, 0.0450, -0.0127],\n", - " [ 0.0028, 0.0185, -0.0092, ..., -0.0441, -0.0428, 0.0232]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.7.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0008, 0.0002, -0.0002, ..., -0.0006, 0.0024, 0.0010],\n", - " [ 0.0004, 0.0005, -0.0004, ..., -0.0005, 0.0018, 0.0012],\n", - " [ 0.0012, -0.0009, 0.0010, ..., -0.0017, 0.0024, -0.0002],\n", - " ...,\n", - " [ 0.0071, -0.0029, -0.0035, ..., -0.0041, 0.0080, -0.0040],\n", - " [ 0.0006, -0.0016, 0.0108, ..., 0.0008, -0.0043, -0.0014],\n", - " [ 0.0016, 0.0023, 0.0044, ..., 0.0002, 0.0046, 0.0030]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.7.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0091, -0.0108, -0.0018, ..., 0.0179, 0.0270, 0.0174],\n", - " [-0.0182, 0.0060, -0.0365, ..., 0.0066, -0.0001, -0.0285],\n", - " [-0.0408, 0.0361, -0.0176, ..., 0.0240, 0.0034, 0.0067],\n", - " ...,\n", - " [ 0.0293, 0.0057, 0.0186, ..., 0.0305, -0.0190, 0.0132],\n", - " [-0.0201, -0.0120, 0.0187, ..., -0.0276, -0.0281, -0.0402],\n", - " [ 0.0082, -0.0213, 0.0019, ..., 0.0298, -0.0016, -0.0259]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.7.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0195, -0.0055, -0.0053, ..., 0.0047, -0.0101, 0.0149],\n", - " [ 0.0014, -0.0081, -0.0072, ..., 0.0070, -0.0058, -0.0003],\n", - " [ 0.0088, 0.0007, 0.0048, ..., -0.0033, 0.0032, 0.0156],\n", - " ...,\n", - " [ 0.0045, -0.0026, -0.0015, ..., 0.0017, -0.0012, 0.0068],\n", - " [ 0.0081, -0.0016, -0.0009, ..., 0.0013, -0.0054, 0.0117],\n", - " [ 0.0039, -0.0103, -0.0033, ..., 0.0092, -0.0028, 0.0041]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.7.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0203, 0.0043, 0.0081, ..., 0.0358, 0.0223, 0.0172],\n", - " [-0.0086, -0.0177, 0.0020, ..., -0.0164, -0.0001, 0.0288],\n", - " [-0.0135, 0.0085, 0.0117, ..., 0.0381, 0.0329, 0.0229],\n", - " ...,\n", - " [-0.0364, -0.0218, -0.0393, ..., 0.0238, 0.0459, 0.0361],\n", - " [ 0.0031, -0.0134, -0.0096, ..., 0.0044, -0.0182, -0.0195],\n", - " [ 0.0105, 0.0036, -0.0233, ..., 0.0360, 0.0076, 0.0006]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.7.layer.1.EncDecAttention.q.lora_B.weight': tensor([[-0.0246, -0.0249, 0.0212, ..., -0.0247, 0.0232, -0.0254],\n", - " [-0.0233, -0.0259, 0.0089, ..., -0.0237, 0.0226, -0.0243],\n", - " [-0.0282, -0.0282, 0.0009, ..., -0.0280, 0.0279, -0.0301],\n", - " ...,\n", - " [-0.0124, -0.0138, 0.0094, ..., -0.0132, 0.0121, -0.0126],\n", - " [ 0.0109, 0.0019, -0.0137, ..., 0.0081, -0.0119, 0.0121],\n", - " [ 0.0128, 0.0076, 0.0082, ..., 0.0131, -0.0137, 0.0123]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.7.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0427, 0.0195, 0.0449, ..., 0.0135, -0.0402, -0.0507],\n", - " [ 0.0125, 0.0196, -0.0151, ..., -0.0420, 0.0035, 0.0245],\n", - " [ 0.0263, -0.0365, 0.0320, ..., 0.0032, -0.0379, 0.0087],\n", - " ...,\n", - " [ 0.0331, 0.0281, -0.0341, ..., 0.0250, 0.0080, 0.0225],\n", - " [-0.0033, -0.0399, -0.0354, ..., 0.0027, 0.0030, 0.0226],\n", - " [-0.0005, 0.0128, 0.0210, ..., -0.0145, 0.0167, -0.0309]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.7.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 8.7193e-03, -9.4752e-03, 4.8379e-03, ..., -6.9801e-03,\n", - " 4.6064e-03, 1.4173e-02],\n", - " [ 1.0323e-02, -7.7941e-03, 1.0070e-02, ..., -2.5394e-03,\n", - " 7.4860e-03, -1.9913e-03],\n", - " [ 2.3713e-03, -3.8002e-03, 6.1123e-03, ..., -6.1484e-03,\n", - " 8.3422e-03, -1.3110e-02],\n", - " ...,\n", - " [ 2.4322e-03, 1.1196e-02, -8.1483e-03, ..., -9.3193e-03,\n", - " -4.1116e-03, 6.0736e-03],\n", - " [-4.6512e-03, 2.2509e-05, 2.8143e-03, ..., 7.7112e-03,\n", - " 1.1012e-02, -1.5901e-02],\n", - " [-2.5353e-02, 1.9855e-02, -1.2999e-02, ..., 2.1589e-02,\n", - " 3.5988e-03, -1.5793e-02]], device='cuda:0'), 'base_model.model.decoder.block.8.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0055, 0.0482, -0.0049, ..., 0.0258, 0.0268, 0.0282],\n", - " [ 0.0004, 0.0061, 0.0378, ..., -0.0061, 0.0131, -0.0106],\n", - " [-0.0127, -0.0368, 0.0114, ..., -0.0245, -0.0108, 0.0082],\n", - " ...,\n", - " [ 0.0270, -0.0469, 0.0392, ..., -0.0057, 0.0246, -0.0449],\n", - " [ 0.0149, -0.0116, 0.0343, ..., 0.0086, -0.0134, -0.0224],\n", - " [-0.0031, -0.0186, 0.0189, ..., 0.0102, 0.0251, 0.0056]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.8.layer.0.SelfAttention.q.lora_B.weight': tensor([[-8.1862e-03, 1.1881e-02, 9.7900e-03, ..., 1.0970e-02,\n", - " -4.8307e-03, 1.3177e-02],\n", - " [ 1.3475e-02, -1.3164e-02, -1.6153e-02, ..., -3.9508e-03,\n", - " -5.1986e-05, -6.6920e-03],\n", - " [ 1.3827e-03, 3.2577e-03, -4.9111e-03, ..., 5.6041e-03,\n", - " 3.4114e-03, -1.3535e-03],\n", - " ...,\n", - " [-2.6834e-02, 2.3956e-02, 2.7925e-02, ..., 2.6604e-02,\n", - " 2.5881e-02, -4.2479e-03],\n", - " [ 9.8006e-03, 5.8498e-03, -1.5935e-02, ..., -1.2355e-02,\n", - " -2.2700e-03, 1.5965e-02],\n", - " [ 1.9455e-02, -1.4621e-02, -1.9711e-02, ..., -2.1926e-02,\n", - " -1.4545e-02, -2.0314e-03]], device='cuda:0'), 'base_model.model.decoder.block.8.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0086, -0.0127, 0.0164, ..., -0.0085, 0.0160, 0.0239],\n", - " [-0.0253, 0.0110, -0.0184, ..., 0.0062, -0.0089, 0.0040],\n", - " [ 0.0300, -0.0046, -0.0077, ..., 0.0109, 0.0151, 0.0012],\n", - " ...,\n", - " [ 0.0069, -0.0402, 0.0182, ..., -0.0243, 0.0367, 0.0397],\n", - " [ 0.0392, -0.0008, 0.0079, ..., -0.0269, -0.0036, -0.0002],\n", - " [ 0.0118, -0.0075, -0.0023, ..., -0.0224, -0.0161, 0.0017]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.8.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0047, 0.0029, -0.0069, ..., -0.0018, -0.0063, -0.0014],\n", - " [ 0.0082, -0.0077, 0.0052, ..., 0.0103, 0.0080, 0.0121],\n", - " [-0.0195, 0.0196, -0.0149, ..., -0.0190, -0.0054, -0.0194],\n", - " ...,\n", - " [-0.0066, 0.0018, -0.0003, ..., -0.0015, -0.0070, 0.0014],\n", - " [ 0.0169, -0.0169, 0.0163, ..., 0.0180, -0.0044, 0.0176],\n", - " [ 0.0115, -0.0148, 0.0143, ..., 0.0089, 0.0165, 0.0110]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.8.layer.1.EncDecAttention.q.lora_A.weight': tensor([[ 0.0088, -0.0198, 0.0352, ..., 0.0017, -0.0137, 0.0235],\n", - " [ 0.0074, 0.0117, 0.0034, ..., -0.0414, -0.0173, -0.0157],\n", - " [-0.0269, 0.0180, 0.0023, ..., -0.0271, -0.0041, -0.0395],\n", - " ...,\n", - " [-0.0052, -0.0061, -0.0074, ..., -0.0023, 0.0183, 0.0248],\n", - " [-0.0085, 0.0143, 0.0291, ..., -0.0317, 0.0161, -0.0440],\n", - " [-0.0242, 0.0251, 0.0268, ..., -0.0222, 0.0009, 0.0223]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.8.layer.1.EncDecAttention.q.lora_B.weight': tensor([[-0.0070, -0.0066, -0.0060, ..., 0.0057, -0.0022, -0.0063],\n", - " [ 0.0075, 0.0020, 0.0047, ..., -0.0082, -0.0012, 0.0090],\n", - " [-0.0149, -0.0074, -0.0133, ..., 0.0114, 0.0035, -0.0127],\n", - " ...,\n", - " [-0.0060, -0.0045, -0.0088, ..., 0.0093, -0.0050, -0.0093],\n", - " [-0.0044, 0.0054, -0.0040, ..., 0.0045, 0.0142, -0.0048],\n", - " [ 0.0050, 0.0038, 0.0082, ..., -0.0098, -0.0018, 0.0068]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.8.layer.1.EncDecAttention.v.lora_A.weight': tensor([[ 0.0147, 0.0348, -0.0170, ..., -0.0441, 0.0298, 0.0520],\n", - " [-0.0203, 0.0083, 0.0499, ..., 0.0094, 0.0094, -0.0457],\n", - " [ 0.0123, 0.0063, 0.0027, ..., 0.0321, -0.0489, -0.0257],\n", - " ...,\n", - " [ 0.0059, 0.0268, -0.0280, ..., 0.0138, -0.0344, -0.0239],\n", - " [ 0.0025, -0.0047, -0.0095, ..., 0.0065, -0.0235, 0.0059],\n", - " [-0.0239, -0.0193, -0.0089, ..., -0.0353, -0.0026, 0.0523]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.8.layer.1.EncDecAttention.v.lora_B.weight': tensor([[-0.0121, 0.0075, 0.0079, ..., 0.0105, 0.0129, -0.0121],\n", - " [ 0.0100, 0.0012, -0.0099, ..., -0.0121, -0.0096, 0.0088],\n", - " [ 0.0122, -0.0076, -0.0110, ..., -0.0120, -0.0115, 0.0121],\n", - " ...,\n", - " [ 0.0093, -0.0080, -0.0108, ..., -0.0075, -0.0032, -0.0013],\n", - " [-0.0115, 0.0213, 0.0142, ..., 0.0134, 0.0117, -0.0052],\n", - " [-0.0036, 0.0162, 0.0016, ..., 0.0018, -0.0023, 0.0021]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.9.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0154, 0.0037, 0.0349, ..., -0.0158, 0.0038, -0.0385],\n", - " [-0.0073, 0.0473, -0.0104, ..., -0.0131, -0.0052, -0.0014],\n", - " [ 0.0135, -0.0418, 0.0044, ..., 0.0084, 0.0183, -0.0136],\n", - " ...,\n", - " [ 0.0269, 0.0013, 0.0371, ..., -0.0096, 0.0070, -0.0114],\n", - " [-0.0020, -0.0478, 0.0082, ..., -0.0092, 0.0111, -0.0221],\n", - " [ 0.0107, -0.0073, -0.0224, ..., 0.0339, -0.0262, 0.0477]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.9.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0193, 0.0199, -0.0044, ..., -0.0231, -0.0193, 0.0192],\n", - " [-0.0032, 0.0015, -0.0103, ..., 0.0019, 0.0071, 0.0039],\n", - " [-0.0046, 0.0031, -0.0092, ..., -0.0005, 0.0053, 0.0053],\n", - " ...,\n", - " [-0.0172, 0.0136, 0.0109, ..., -0.0211, -0.0160, 0.0031],\n", - " [-0.0070, 0.0049, -0.0063, ..., -0.0058, -0.0043, 0.0051],\n", - " [-0.0118, 0.0162, -0.0009, ..., -0.0190, -0.0135, 0.0053]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.9.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0438, -0.0172, 0.0363, ..., 0.0086, -0.0087, 0.0066],\n", - " [ 0.0410, -0.0098, 0.0053, ..., -0.0250, 0.0264, 0.0233],\n", - " [ 0.0430, -0.0061, 0.0150, ..., 0.0156, 0.0056, -0.0025],\n", - " ...,\n", - " [ 0.0078, -0.0022, 0.0056, ..., -0.0328, 0.0030, -0.0042],\n", - " [ 0.0094, -0.0351, -0.0269, ..., -0.0067, 0.0339, 0.0215],\n", - " [ 0.0293, 0.0221, 0.0090, ..., 0.0187, 0.0156, 0.0453]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.9.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0210, -0.0218, -0.0219, ..., 0.0225, -0.0235, -0.0230],\n", - " [-0.0032, 0.0079, -0.0071, ..., -0.0083, 0.0067, 0.0079],\n", - " [ 0.0085, -0.0037, -0.0048, ..., 0.0048, -0.0062, -0.0076],\n", - " ...,\n", - " [-0.0054, 0.0032, 0.0100, ..., -0.0020, 0.0055, 0.0025],\n", - " [-0.0110, 0.0081, 0.0158, ..., -0.0079, 0.0112, 0.0075],\n", - " [-0.0049, 0.0053, 0.0072, ..., -0.0049, 0.0068, 0.0042]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.9.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0071, -0.0169, -0.0282, ..., 0.0228, 0.0082, -0.0134],\n", - " [-0.0093, -0.0039, -0.0467, ..., 0.0432, 0.0402, 0.0078],\n", - " [ 0.0147, 0.0395, 0.0144, ..., -0.0338, -0.0109, 0.0084],\n", - " ...,\n", - " [ 0.0166, -0.0008, 0.0265, ..., -0.0120, -0.0385, -0.0455],\n", - " [ 0.0310, -0.0093, 0.0134, ..., -0.0040, -0.0034, -0.0137],\n", - " [-0.0219, -0.0405, -0.0077, ..., 0.0289, -0.0082, -0.0013]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.9.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0050, -0.0137, 0.0062, ..., 0.0020, -0.0027, -0.0102],\n", - " [-0.0208, 0.0056, -0.0130, ..., -0.0062, 0.0127, 0.0047],\n", - " [-0.0034, -0.0049, 0.0036, ..., 0.0025, 0.0036, 0.0013],\n", - " ...,\n", - " [ 0.0123, -0.0268, 0.0155, ..., 0.0150, -0.0170, -0.0189],\n", - " [-0.0296, 0.0265, -0.0202, ..., -0.0150, 0.0275, 0.0163],\n", - " [-0.0052, 0.0020, -0.0008, ..., -0.0013, 0.0061, 0.0013]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.9.layer.1.EncDecAttention.v.lora_A.weight': tensor([[ 0.0483, 0.0039, 0.0312, ..., -0.0194, 0.0217, 0.0157],\n", - " [-0.0274, 0.0153, 0.0293, ..., 0.0371, -0.0078, -0.0204],\n", - " [-0.0226, 0.0241, -0.0100, ..., 0.0308, -0.0354, -0.0683],\n", - " ...,\n", - " [ 0.0043, 0.0249, -0.0220, ..., -0.0079, 0.0153, 0.0055],\n", - " [ 0.0618, 0.0091, 0.0140, ..., -0.0128, 0.0089, 0.0166],\n", - " [-0.0502, -0.0238, 0.0167, ..., 0.0662, 0.0163, -0.0201]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.9.layer.1.EncDecAttention.v.lora_B.weight': tensor([[-0.0136, 0.0160, 0.0156, ..., -0.0165, -0.0171, 0.0161],\n", - " [ 0.0025, -0.0026, -0.0052, ..., 0.0055, 0.0045, -0.0020],\n", - " [-0.0057, 0.0066, 0.0048, ..., -0.0033, -0.0056, 0.0068],\n", - " ...,\n", - " [ 0.0240, -0.0242, -0.0254, ..., 0.0231, 0.0241, -0.0242],\n", - " [ 0.0213, -0.0170, -0.0214, ..., 0.0198, 0.0180, -0.0202],\n", - " [-0.0150, 0.0218, 0.0179, ..., -0.0128, -0.0194, 0.0167]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.10.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0025, -0.0290, 0.0255, ..., -0.0181, 0.0138, 0.0025],\n", - " [-0.0440, -0.0035, 0.0296, ..., 0.0243, 0.0088, -0.0053],\n", - " [-0.0272, 0.0054, -0.0146, ..., 0.0257, 0.0243, 0.0020],\n", - " ...,\n", - " [ 0.0010, 0.0344, 0.0134, ..., 0.0117, -0.0007, -0.0075],\n", - " [ 0.0423, -0.0215, 0.0008, ..., -0.0328, -0.0238, -0.0170],\n", - " [-0.0181, 0.0097, 0.0098, ..., 0.0165, -0.0156, -0.0103]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.10.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0063, 0.0127, 0.0105, ..., 0.0136, 0.0137, 0.0123],\n", - " [ 0.0126, -0.0181, -0.0162, ..., -0.0186, -0.0186, -0.0181],\n", - " [-0.0070, 0.0156, 0.0164, ..., 0.0148, 0.0144, 0.0143],\n", - " ...,\n", - " [-0.0039, 0.0014, 0.0003, ..., 0.0025, -0.0060, 0.0027],\n", - " [ 0.0025, 0.0164, 0.0130, ..., 0.0138, -0.0010, 0.0125],\n", - " [ 0.0102, 0.0055, 0.0062, ..., -0.0012, -0.0060, 0.0018]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.10.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0024, -0.0390, -0.0036, ..., 0.0109, -0.0263, 0.0339],\n", - " [ 0.0104, -0.0131, 0.0153, ..., -0.0151, 0.0249, 0.0227],\n", - " [-0.0228, 0.0018, 0.0311, ..., -0.0179, 0.0120, -0.0137],\n", - " ...,\n", - " [ 0.0398, -0.0251, 0.0051, ..., 0.0207, 0.0257, 0.0066],\n", - " [ 0.0026, 0.0058, -0.0247, ..., 0.0021, 0.0233, -0.0101],\n", - " [ 0.0265, 0.0026, 0.0404, ..., 0.0282, -0.0161, 0.0450]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.10.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0041, 0.0035, -0.0041, ..., 0.0040, 0.0048, 0.0029],\n", - " [-0.0153, -0.0180, 0.0142, ..., -0.0159, -0.0138, -0.0151],\n", - " [ 0.0144, 0.0131, -0.0153, ..., 0.0147, 0.0151, 0.0122],\n", - " ...,\n", - " [-0.0130, -0.0103, 0.0154, ..., -0.0113, -0.0158, -0.0153],\n", - " [-0.0170, -0.0153, 0.0144, ..., -0.0125, -0.0155, -0.0185],\n", - " [-0.0158, -0.0154, 0.0174, ..., -0.0161, -0.0178, -0.0153]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.10.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0304, -0.0075, -0.0230, ..., 0.0063, -0.0018, 0.0278],\n", - " [-0.0076, -0.0079, 0.0015, ..., -0.0316, 0.0023, 0.0243],\n", - " [ 0.0084, 0.0124, 0.0012, ..., 0.0109, -0.0317, 0.0220],\n", - " ...,\n", - " [-0.0283, 0.0414, -0.0127, ..., -0.0159, 0.0182, 0.0143],\n", - " [-0.0043, -0.0115, -0.0229, ..., 0.0092, -0.0283, 0.0275],\n", - " [-0.0176, 0.0361, -0.0088, ..., -0.0113, -0.0109, 0.0135]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.10.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 8.9059e-03, 5.9997e-03, 3.2296e-03, ..., 4.1912e-03,\n", - " -7.3702e-04, 3.2630e-03],\n", - " [ 1.4178e-02, 1.5074e-02, 8.4570e-03, ..., 3.8711e-03,\n", - " 1.0596e-02, 6.3549e-03],\n", - " [-1.2284e-02, -9.7453e-03, 5.3009e-04, ..., 8.9120e-03,\n", - " 5.1250e-04, -5.0001e-03],\n", - " ...,\n", - " [ 4.1886e-03, 8.0591e-05, -2.7274e-04, ..., 8.2174e-03,\n", - " -2.2814e-03, 4.5755e-04],\n", - " [ 1.7286e-02, 1.7134e-02, 1.1415e-02, ..., 2.6513e-02,\n", - " 2.5976e-03, 1.1181e-02],\n", - " [ 1.2990e-02, 1.4693e-02, 1.0798e-02, ..., 1.6177e-02,\n", - " 5.8300e-03, 5.1070e-03]], device='cuda:0'), 'base_model.model.decoder.block.10.layer.1.EncDecAttention.v.lora_A.weight': tensor([[ 0.0020, -0.0097, -0.0073, ..., -0.0110, 0.0074, 0.0097],\n", - " [-0.0227, 0.0208, -0.0257, ..., -0.0031, -0.0147, -0.0224],\n", - " [-0.0020, 0.0450, 0.0228, ..., -0.0019, -0.0001, 0.0342],\n", - " ...,\n", - " [-0.0067, -0.0084, 0.0036, ..., 0.0493, -0.0207, -0.0334],\n", - " [ 0.0320, -0.0389, -0.0380, ..., 0.0327, -0.0444, 0.0107],\n", - " [-0.0115, -0.0011, 0.0172, ..., 0.0082, -0.0172, -0.0654]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.10.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 0.0213, 0.0044, 0.0228, ..., -0.0223, -0.0208, -0.0105],\n", - " [-0.0181, 0.0040, -0.0209, ..., 0.0195, 0.0192, 0.0131],\n", - " [ 0.0149, 0.0117, 0.0133, ..., -0.0163, -0.0158, -0.0026],\n", - " ...,\n", - " [ 0.0104, 0.0159, 0.0134, ..., -0.0119, -0.0154, -0.0064],\n", - " [-0.0195, -0.0169, -0.0204, ..., 0.0204, 0.0179, 0.0020],\n", - " [-0.0017, 0.0049, -0.0006, ..., -0.0002, 0.0053, 0.0112]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.11.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0113, -0.0173, -0.0027, ..., 0.0034, 0.0462, -0.0084],\n", - " [-0.0225, -0.0064, -0.0297, ..., -0.0134, 0.0226, 0.0151],\n", - " [ 0.0253, -0.0319, 0.0379, ..., 0.0115, 0.0016, -0.0272],\n", - " ...,\n", - " [ 0.0177, 0.0429, -0.0232, ..., 0.0427, 0.0081, 0.0265],\n", - " [-0.0095, -0.0153, -0.0117, ..., -0.0021, -0.0487, -0.0410],\n", - " [ 0.0021, 0.0395, 0.0165, ..., 0.0390, 0.0358, 0.0250]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.11.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0042, -0.0051, 0.0075, ..., -0.0086, 0.0048, -0.0102],\n", - " [-0.0015, -0.0072, 0.0053, ..., -0.0051, 0.0027, -0.0075],\n", - " [ 0.0172, 0.0165, -0.0048, ..., 0.0060, -0.0144, 0.0042],\n", - " ...,\n", - " [-0.0212, -0.0192, 0.0205, ..., -0.0172, 0.0160, -0.0143],\n", - " [ 0.0145, 0.0102, -0.0123, ..., 0.0128, -0.0037, 0.0073],\n", - " [-0.0072, -0.0060, 0.0064, ..., -0.0072, 0.0052, -0.0087]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.11.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0115, -0.0157, 0.0200, ..., -0.0234, 0.0127, 0.0111],\n", - " [ 0.0194, 0.0063, -0.0089, ..., -0.0268, -0.0184, -0.0171],\n", - " [ 0.0094, -0.0173, 0.0352, ..., -0.0146, -0.0239, 0.0131],\n", - " ...,\n", - " [ 0.0015, -0.0065, -0.0278, ..., -0.0262, 0.0022, -0.0093],\n", - " [ 0.0198, -0.0332, -0.0144, ..., 0.0026, 0.0404, 0.0179],\n", - " [ 0.0105, -0.0364, 0.0262, ..., -0.0171, 0.0403, 0.0121]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.11.layer.0.SelfAttention.v.lora_B.weight': tensor([[-9.5707e-03, -1.2350e-02, 4.2512e-03, ..., -6.5697e-03,\n", - " 1.0621e-02, 8.1075e-03],\n", - " [-1.3202e-03, 1.8115e-02, -1.6808e-02, ..., 1.9649e-02,\n", - " -1.0918e-02, -1.9214e-02],\n", - " [ 1.8847e-02, -1.7127e-02, 1.7074e-02, ..., -1.9088e-02,\n", - " 8.5360e-03, 1.7337e-02],\n", - " ...,\n", - " [-2.9112e-03, -2.2279e-02, 2.3047e-02, ..., -2.6989e-02,\n", - " 2.1065e-02, 2.5149e-02],\n", - " [ 1.2249e-02, -5.4296e-04, 1.3645e-02, ..., -7.0064e-04,\n", - " -1.2102e-03, -8.9333e-05],\n", - " [-3.2261e-03, 1.1069e-02, -3.9940e-03, ..., 1.3339e-02,\n", - " -1.5026e-02, -1.2184e-02]], device='cuda:0'), 'base_model.model.decoder.block.11.layer.1.EncDecAttention.q.lora_A.weight': tensor([[ 0.0151, 0.0504, 0.0151, ..., -0.0405, -0.0372, 0.0045],\n", - " [ 0.0058, -0.0132, -0.0114, ..., 0.0315, -0.0070, -0.0143],\n", - " [ 0.0007, -0.0070, -0.0318, ..., -0.0215, 0.0071, 0.0102],\n", - " ...,\n", - " [-0.0004, 0.0088, 0.0068, ..., 0.0079, -0.0094, 0.0206],\n", - " [-0.0109, 0.0159, -0.0031, ..., -0.0096, -0.0329, -0.0108],\n", - " [-0.0080, 0.0129, 0.0347, ..., -0.0115, 0.0031, 0.0019]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.11.layer.1.EncDecAttention.q.lora_B.weight': tensor([[-0.0131, 0.0158, 0.0049, ..., 0.0131, -0.0112, -0.0145],\n", - " [ 0.0174, -0.0180, -0.0168, ..., -0.0093, 0.0182, 0.0186],\n", - " [-0.0105, 0.0116, 0.0139, ..., -0.0073, -0.0136, -0.0119],\n", - " ...,\n", - " [-0.0058, 0.0093, 0.0242, ..., -0.0101, -0.0032, 0.0060],\n", - " [ 0.0191, -0.0159, -0.0204, ..., 0.0079, 0.0200, 0.0092],\n", - " [ 0.0009, -0.0016, 0.0067, ..., 0.0027, -0.0018, 0.0032]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.11.layer.1.EncDecAttention.v.lora_A.weight': tensor([[ 0.0233, 0.0394, -0.0297, ..., 0.0140, -0.0410, -0.0096],\n", - " [-0.0128, -0.0103, 0.0050, ..., -0.0203, 0.0273, -0.0021],\n", - " [ 0.0241, 0.0009, -0.0340, ..., -0.0166, -0.0095, 0.0592],\n", - " ...,\n", - " [ 0.0344, 0.0115, -0.0103, ..., -0.0338, 0.0280, 0.0553],\n", - " [-0.0379, -0.0005, 0.0447, ..., -0.0204, 0.0497, -0.0573],\n", - " [ 0.0310, -0.0138, -0.0048, ..., 0.0067, -0.0408, 0.0392]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.11.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 2.2074e-02, -2.3050e-02, -2.0557e-02, ..., -3.0574e-02,\n", - " 6.5095e-04, 7.4577e-03],\n", - " [ 1.9601e-02, -1.9351e-02, -1.3357e-02, ..., -1.9333e-02,\n", - " -5.8716e-03, 1.7343e-02],\n", - " [-2.0707e-02, 2.1192e-02, 1.7973e-02, ..., 2.2283e-02,\n", - " 7.3185e-03, -1.4129e-02],\n", - " ...,\n", - " [-1.2650e-02, 5.2220e-03, 2.7920e-04, ..., 1.1320e-03,\n", - " 2.8257e-03, -8.0793e-03],\n", - " [-9.3410e-03, -1.6412e-03, -1.1426e-02, ..., -1.1142e-02,\n", - " 5.2212e-03, -1.3261e-03],\n", - " [-1.8118e-02, 1.8327e-02, -1.9824e-05, ..., 2.1927e-02,\n", - " 8.2317e-03, -1.1294e-02]], device='cuda:0'), 'base_model.model.decoder.block.12.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0103, 0.0235, -0.0445, ..., -0.0011, 0.0211, 0.0054],\n", - " [ 0.0231, 0.0272, -0.0111, ..., -0.0416, -0.0326, -0.0198],\n", - " [-0.0178, -0.0060, 0.0432, ..., -0.0134, 0.0074, 0.0074],\n", - " ...,\n", - " [ 0.0253, 0.0366, 0.0049, ..., -0.0379, -0.0321, 0.0107],\n", - " [-0.0126, -0.0017, -0.0359, ..., 0.0121, 0.0077, 0.0259],\n", - " [ 0.0278, 0.0117, -0.0050, ..., 0.0169, 0.0003, 0.0109]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.12.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0189, -0.0169, -0.0191, ..., -0.0217, 0.0187, -0.0174],\n", - " [-0.0155, 0.0188, 0.0232, ..., 0.0169, -0.0256, 0.0255],\n", - " [-0.0193, 0.0153, 0.0159, ..., 0.0276, -0.0155, 0.0142],\n", - " ...,\n", - " [-0.0200, 0.0177, 0.0180, ..., 0.0193, -0.0177, 0.0189],\n", - " [-0.0146, 0.0139, 0.0137, ..., 0.0132, -0.0135, 0.0142],\n", - " [ 0.0102, -0.0118, -0.0130, ..., -0.0131, 0.0132, -0.0101]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.12.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 2.4031e-03, 2.1623e-02, 9.9847e-03, ..., 1.5630e-02,\n", - " -8.4626e-03, -1.9406e-02],\n", - " [ 6.6478e-03, 2.4983e-02, -1.5062e-02, ..., 1.9716e-02,\n", - " -1.7178e-02, 1.3483e-03],\n", - " [ 9.8836e-03, 6.9536e-03, 2.2963e-02, ..., 3.3474e-02,\n", - " 2.1320e-02, 3.9549e-02],\n", - " ...,\n", - " [-2.3796e-02, -7.5400e-06, 3.0448e-02, ..., 4.2281e-02,\n", - " 2.4435e-02, 3.4439e-03],\n", - " [-8.6579e-03, 2.4908e-02, -1.4695e-02, ..., -1.7937e-02,\n", - " -2.9916e-02, -7.0838e-03],\n", - " [-1.8854e-02, -4.6361e-02, 4.9669e-02, ..., -1.4553e-04,\n", - " 2.3075e-02, 1.9088e-03]], device='cuda:0'), 'base_model.model.decoder.block.12.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0074, -0.0017, 0.0112, ..., 0.0109, -0.0103, 0.0119],\n", - " [-0.0149, -0.0008, 0.0157, ..., 0.0172, -0.0093, 0.0181],\n", - " [ 0.0007, 0.0051, -0.0025, ..., -0.0014, 0.0031, -0.0015],\n", - " ...,\n", - " [-0.0013, -0.0033, -0.0026, ..., -0.0015, -0.0034, -0.0038],\n", - " [-0.0056, -0.0132, 0.0050, ..., 0.0036, -0.0130, -0.0087],\n", - " [-0.0098, -0.0120, 0.0107, ..., 0.0110, -0.0117, 0.0023]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.12.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0046, 0.0211, 0.0200, ..., -0.0087, 0.0239, 0.0120],\n", - " [-0.0436, -0.0509, -0.0096, ..., -0.0034, 0.0071, -0.0181],\n", - " [-0.0019, -0.0168, -0.0081, ..., -0.0142, -0.0002, 0.0396],\n", - " ...,\n", - " [ 0.0069, -0.0059, 0.0063, ..., -0.0305, 0.0049, 0.0234],\n", - " [-0.0008, -0.0510, -0.0299, ..., 0.0231, 0.0161, 0.0187],\n", - " [ 0.0062, -0.0050, -0.0067, ..., 0.0040, -0.0048, -0.0111]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.12.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0257, -0.0312, -0.0249, ..., -0.0225, -0.0249, 0.0227],\n", - " [ 0.0275, -0.0350, -0.0293, ..., -0.0257, -0.0283, 0.0259],\n", - " [ 0.0032, -0.0025, 0.0020, ..., -0.0031, -0.0025, 0.0046],\n", - " ...,\n", - " [-0.0013, 0.0057, 0.0068, ..., 0.0017, 0.0016, -0.0019],\n", - " [ 0.0046, -0.0059, -0.0001, ..., -0.0073, -0.0064, 0.0085],\n", - " [-0.0058, 0.0037, 0.0097, ..., 0.0026, 0.0047, -0.0013]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.12.layer.1.EncDecAttention.v.lora_A.weight': tensor([[ 0.0103, -0.0327, -0.0249, ..., 0.0200, -0.0102, 0.0046],\n", - " [ 0.0150, 0.0122, -0.0172, ..., 0.0032, 0.0125, -0.0109],\n", - " [-0.0103, 0.0146, 0.0295, ..., -0.0434, -0.0173, -0.0248],\n", - " ...,\n", - " [ 0.0349, -0.0145, 0.0197, ..., 0.0345, -0.0413, 0.0070],\n", - " [-0.0015, -0.0148, 0.0261, ..., -0.0024, -0.0134, -0.0016],\n", - " [ 0.0220, 0.0160, 0.0229, ..., -0.0040, -0.0127, 0.0081]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.12.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 0.0108, 0.0126, -0.0129, ..., 0.0061, -0.0068, 0.0116],\n", - " [-0.0031, -0.0131, 0.0072, ..., -0.0232, 0.0122, -0.0168],\n", - " [-0.0215, -0.0097, 0.0125, ..., -0.0147, 0.0112, -0.0142],\n", - " ...,\n", - " [-0.0178, -0.0196, 0.0217, ..., -0.0212, 0.0185, -0.0224],\n", - " [ 0.0107, -0.0059, 0.0002, ..., -0.0074, 0.0075, -0.0123],\n", - " [ 0.0042, -0.0160, -0.0046, ..., -0.0154, 0.0115, -0.0159]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.13.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0219, 0.0094, -0.0227, ..., -0.0118, -0.0069, -0.0019],\n", - " [ 0.0170, 0.0005, 0.0317, ..., -0.0442, -0.0148, -0.0178],\n", - " [-0.0057, -0.0064, 0.0218, ..., -0.0492, -0.0141, -0.0176],\n", - " ...,\n", - " [ 0.0341, -0.0037, 0.0090, ..., 0.0453, 0.0237, 0.0424],\n", - " [ 0.0107, -0.0036, 0.0088, ..., -0.0421, -0.0325, -0.0168],\n", - " [ 0.0242, -0.0061, -0.0386, ..., -0.0047, -0.0117, -0.0059]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.13.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0086, -0.0127, -0.0096, ..., 0.0039, -0.0092, 0.0090],\n", - " [ 0.0119, -0.0163, -0.0191, ..., 0.0047, -0.0144, 0.0121],\n", - " [-0.0091, 0.0139, 0.0143, ..., -0.0050, 0.0106, -0.0094],\n", - " ...,\n", - " [ 0.0141, -0.0123, -0.0156, ..., 0.0110, -0.0122, 0.0114],\n", - " [-0.0024, 0.0034, 0.0031, ..., -0.0013, 0.0014, -0.0071],\n", - " [ 0.0017, -0.0023, -0.0021, ..., -0.0023, 0.0005, 0.0037]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.13.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0047, 0.0136, -0.0285, ..., -0.0433, 0.0030, 0.0144],\n", - " [ 0.0114, 0.0227, -0.0303, ..., 0.0104, 0.0045, 0.0162],\n", - " [ 0.0358, 0.0059, 0.0081, ..., -0.0286, -0.0211, -0.0286],\n", - " ...,\n", - " [ 0.0013, -0.0096, 0.0008, ..., 0.0270, -0.0111, 0.0180],\n", - " [ 0.0205, 0.0279, -0.0179, ..., -0.0381, -0.0168, -0.0347],\n", - " [ 0.0088, -0.0382, 0.0431, ..., 0.0359, -0.0130, 0.0356]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.13.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0092, -0.0092, -0.0036, ..., 0.0120, -0.0077, 0.0126],\n", - " [ 0.0039, 0.0076, 0.0078, ..., 0.0046, 0.0068, 0.0035],\n", - " [-0.0306, -0.0290, -0.0167, ..., 0.0265, -0.0283, 0.0298],\n", - " ...,\n", - " [ 0.0154, 0.0149, 0.0118, ..., -0.0089, 0.0155, -0.0134],\n", - " [-0.0122, -0.0120, -0.0107, ..., 0.0109, -0.0126, 0.0120],\n", - " [-0.0047, -0.0033, 0.0010, ..., 0.0054, 0.0011, 0.0038]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.13.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0253, -0.0134, -0.0078, ..., -0.0472, -0.0238, -0.0436],\n", - " [ 0.0033, -0.0051, 0.0008, ..., 0.0166, -0.0053, 0.0373],\n", - " [-0.0360, -0.0351, -0.0227, ..., 0.0108, 0.0331, -0.0146],\n", - " ...,\n", - " [-0.0030, 0.0173, -0.0300, ..., 0.0246, 0.0367, 0.0359],\n", - " [ 0.0126, 0.0130, 0.0141, ..., 0.0443, -0.0219, -0.0240],\n", - " [ 0.0219, 0.0208, -0.0084, ..., 0.0185, 0.0198, 0.0202]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.13.layer.1.EncDecAttention.q.lora_B.weight': tensor([[-0.0225, 0.0218, 0.0237, ..., 0.0197, 0.0006, 0.0226],\n", - " [-0.0211, 0.0232, 0.0246, ..., 0.0203, 0.0224, 0.0244],\n", - " [-0.0058, -0.0001, 0.0041, ..., 0.0010, 0.0181, 0.0170],\n", - " ...,\n", - " [ 0.0031, 0.0004, 0.0067, ..., -0.0013, 0.0070, 0.0063],\n", - " [ 0.0117, -0.0107, -0.0180, ..., -0.0149, -0.0202, -0.0204],\n", - " [-0.0079, 0.0058, 0.0041, ..., 0.0028, 0.0070, 0.0060]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.13.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0245, -0.0014, 0.0118, ..., 0.0183, 0.0520, 0.0118],\n", - " [-0.0252, 0.0178, 0.0142, ..., 0.0115, 0.0291, 0.0260],\n", - " [-0.0256, 0.0298, 0.0250, ..., -0.0366, 0.0294, 0.0026],\n", - " ...,\n", - " [ 0.0112, 0.0158, -0.0059, ..., 0.0497, -0.0213, -0.0500],\n", - " [-0.0259, 0.0207, -0.0143, ..., -0.0326, 0.0212, 0.0302],\n", - " [-0.0123, 0.0010, -0.0071, ..., 0.0158, -0.0003, -0.0120]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.13.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 0.0100, 0.0163, 0.0209, ..., -0.0165, 0.0191, 0.0183],\n", - " [-0.0046, -0.0019, -0.0039, ..., 0.0066, -0.0077, -0.0095],\n", - " [ 0.0071, 0.0146, 0.0189, ..., -0.0203, 0.0199, 0.0203],\n", - " ...,\n", - " [ 0.0047, 0.0073, 0.0046, ..., -0.0153, 0.0060, 0.0022],\n", - " [-0.0199, -0.0175, -0.0212, ..., 0.0074, -0.0213, -0.0187],\n", - " [-0.0100, -0.0072, -0.0100, ..., 0.0194, -0.0130, -0.0081]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.14.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0374, -0.0102, -0.0033, ..., 0.0444, -0.0081, -0.0211],\n", - " [ 0.0251, -0.0035, -0.0038, ..., -0.0019, -0.0239, 0.0165],\n", - " [ 0.0151, 0.0200, -0.0244, ..., 0.0094, -0.0323, -0.0069],\n", - " ...,\n", - " [ 0.0082, 0.0296, -0.0086, ..., 0.0067, -0.0011, 0.0200],\n", - " [ 0.0051, -0.0162, 0.0073, ..., -0.0121, -0.0001, -0.0147],\n", - " [-0.0156, 0.0238, 0.0216, ..., -0.0370, -0.0026, -0.0052]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.14.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0076, -0.0004, -0.0016, ..., 0.0082, 0.0001, 0.0090],\n", - " [-0.0125, -0.0170, -0.0166, ..., -0.0114, 0.0149, -0.0104],\n", - " [ 0.0138, 0.0154, 0.0162, ..., 0.0136, -0.0150, 0.0128],\n", - " ...,\n", - " [ 0.0039, 0.0059, 0.0069, ..., 0.0038, -0.0016, 0.0073],\n", - " [ 0.0080, -0.0075, -0.0050, ..., -0.0102, 0.0071, -0.0042],\n", - " [ 0.0025, 0.0050, 0.0025, ..., 0.0080, -0.0047, 0.0008]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.14.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0017, 0.0335, 0.0202, ..., -0.0248, 0.0198, -0.0257],\n", - " [ 0.0159, -0.0008, 0.0156, ..., 0.0062, 0.0148, 0.0142],\n", - " [ 0.0241, 0.0451, -0.0204, ..., -0.0211, -0.0325, -0.0450],\n", - " ...,\n", - " [ 0.0282, 0.0004, -0.0164, ..., -0.0083, -0.0320, 0.0056],\n", - " [-0.0371, -0.0190, -0.0167, ..., 0.0097, 0.0287, 0.0375],\n", - " [ 0.0385, 0.0255, -0.0023, ..., -0.0265, -0.0068, -0.0086]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.14.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0089, -0.0147, -0.0117, ..., -0.0144, 0.0151, -0.0129],\n", - " [ 0.0181, 0.0294, 0.0288, ..., 0.0267, -0.0299, 0.0301],\n", - " [-0.0206, -0.0198, -0.0207, ..., -0.0200, 0.0170, -0.0197],\n", - " ...,\n", - " [-0.0094, 0.0028, 0.0006, ..., 0.0009, -0.0027, -0.0012],\n", - " [-0.0130, -0.0167, -0.0158, ..., -0.0158, 0.0143, -0.0159],\n", - " [-0.0003, 0.0117, 0.0091, ..., 0.0089, -0.0094, 0.0117]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.14.layer.1.EncDecAttention.q.lora_A.weight': tensor([[ 0.0170, -0.0165, -0.0077, ..., 0.0260, 0.0332, -0.0158],\n", - " [ 0.0055, -0.0062, 0.0312, ..., 0.0198, 0.0379, -0.0031],\n", - " [-0.0055, -0.0003, -0.0136, ..., 0.0274, 0.0339, 0.0083],\n", - " ...,\n", - " [-0.0270, -0.0042, 0.0200, ..., -0.0558, -0.0361, -0.0222],\n", - " [-0.0186, 0.0363, -0.0226, ..., -0.0413, -0.0183, 0.0342],\n", - " [ 0.0233, -0.0100, -0.0039, ..., 0.0253, 0.0414, -0.0166]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.14.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0069, 0.0116, 0.0013, ..., -0.0092, -0.0087, 0.0109],\n", - " [-0.0145, -0.0106, -0.0112, ..., 0.0108, 0.0059, -0.0067],\n", - " [ 0.0009, 0.0047, 0.0026, ..., -0.0067, -0.0045, 0.0030],\n", - " ...,\n", - " [-0.0020, 0.0029, 0.0040, ..., -0.0054, -0.0019, 0.0033],\n", - " [ 0.0104, 0.0127, 0.0050, ..., -0.0121, -0.0120, 0.0125],\n", - " [ 0.0132, 0.0152, 0.0163, ..., -0.0161, -0.0142, 0.0154]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.14.layer.1.EncDecAttention.v.lora_A.weight': tensor([[ 0.0538, -0.0210, -0.0223, ..., 0.0296, 0.0070, 0.0165],\n", - " [ 0.0071, 0.0046, -0.0163, ..., -0.0276, -0.0093, 0.0149],\n", - " [ 0.0524, 0.0300, -0.0185, ..., 0.0410, -0.0501, -0.0439],\n", - " ...,\n", - " [ 0.0027, -0.0458, 0.0228, ..., -0.0456, 0.0372, -0.0044],\n", - " [-0.0516, -0.0328, 0.0084, ..., -0.0020, 0.0101, -0.0217],\n", - " [-0.0488, -0.0184, -0.0100, ..., -0.0236, -0.0005, 0.0046]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.14.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 0.0304, -0.0266, 0.0291, ..., -0.0293, -0.0279, -0.0280],\n", - " [ 0.0024, -0.0023, 0.0030, ..., -0.0006, 0.0019, -0.0014],\n", - " [-0.0144, 0.0198, -0.0207, ..., 0.0224, 0.0214, 0.0192],\n", - " ...,\n", - " [ 0.0128, -0.0158, 0.0149, ..., -0.0154, -0.0151, -0.0157],\n", - " [ 0.0039, -0.0054, 0.0027, ..., -0.0020, -0.0050, -0.0052],\n", - " [ 0.0104, -0.0127, 0.0108, ..., -0.0100, -0.0103, -0.0124]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.15.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0160, -0.0093, 0.0195, ..., -0.0205, -0.0025, -0.0271],\n", - " [ 0.0169, -0.0051, 0.0047, ..., 0.0031, 0.0108, 0.0349],\n", - " [-0.0346, 0.0079, 0.0012, ..., -0.0336, -0.0087, -0.0240],\n", - " ...,\n", - " [ 0.0279, -0.0023, -0.0044, ..., 0.0205, -0.0074, 0.0415],\n", - " [ 0.0306, -0.0036, -0.0323, ..., -0.0290, 0.0356, 0.0230],\n", - " [-0.0206, 0.0021, 0.0137, ..., -0.0037, 0.0064, -0.0108]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.15.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0021, 0.0034, -0.0017, ..., 0.0019, 0.0005, -0.0005],\n", - " [ 0.0027, 0.0014, 0.0029, ..., -0.0032, -0.0004, 0.0007],\n", - " [ 0.0128, -0.0122, 0.0117, ..., -0.0105, -0.0078, 0.0099],\n", - " ...,\n", - " [ 0.0084, -0.0047, 0.0049, ..., -0.0058, -0.0098, 0.0095],\n", - " [ 0.0076, -0.0041, 0.0042, ..., -0.0087, -0.0084, 0.0071],\n", - " [-0.0067, -0.0047, -0.0006, ..., 0.0078, 0.0089, -0.0042]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.15.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0054, -0.0268, 0.0230, ..., 0.0257, -0.0094, -0.0198],\n", - " [-0.0194, 0.0296, 0.0148, ..., -0.0157, -0.0334, -0.0433],\n", - " [-0.0143, 0.0536, 0.0044, ..., -0.0396, -0.0147, -0.0384],\n", - " ...,\n", - " [-0.0108, 0.0081, 0.0216, ..., -0.0186, -0.0146, 0.0007],\n", - " [-0.0073, -0.0274, -0.0106, ..., -0.0029, -0.0138, 0.0210],\n", - " [-0.0089, -0.0516, 0.0305, ..., 0.0103, 0.0208, -0.0060]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.15.layer.0.SelfAttention.v.lora_B.weight': tensor([[-7.7887e-03, -4.4329e-03, -1.0258e-02, ..., -1.1257e-02,\n", - " 9.5218e-03, 1.0012e-02],\n", - " [-1.3725e-02, -1.3488e-02, -2.1645e-03, ..., -7.9241e-04,\n", - " 2.8600e-03, 2.6073e-03],\n", - " [-1.9941e-02, -5.7291e-03, -3.8434e-03, ..., -4.3191e-03,\n", - " 7.2316e-03, 3.0308e-03],\n", - " ...,\n", - " [-2.7335e-03, -2.2611e-02, -1.1037e-02, ..., -9.8621e-03,\n", - " 1.4747e-02, 1.1001e-02],\n", - " [-1.6099e-02, -1.4242e-03, -2.9456e-03, ..., 6.2675e-04,\n", - " 5.8403e-05, 3.8604e-03],\n", - " [-1.2524e-02, -1.1022e-02, -1.3438e-02, ..., -1.4034e-02,\n", - " 1.6053e-02, 1.5081e-02]], device='cuda:0'), 'base_model.model.decoder.block.15.layer.1.EncDecAttention.q.lora_A.weight': tensor([[ 0.0230, 0.0099, 0.0328, ..., -0.0349, 0.0029, 0.0024],\n", - " [ 0.0306, 0.0177, -0.0057, ..., -0.0538, -0.0250, 0.0284],\n", - " [-0.0387, -0.0495, 0.0077, ..., 0.0302, -0.0180, 0.0018],\n", - " ...,\n", - " [-0.0281, -0.0137, -0.0024, ..., 0.0016, 0.0356, 0.0271],\n", - " [-0.0477, -0.0220, 0.0125, ..., 0.0446, 0.0032, -0.0169],\n", - " [-0.0168, -0.0307, -0.0202, ..., 0.0207, 0.0262, 0.0194]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.15.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0084, 0.0045, -0.0095, ..., -0.0095, -0.0065, -0.0033],\n", - " [ 0.0140, 0.0250, -0.0206, ..., -0.0109, -0.0197, -0.0172],\n", - " [-0.0027, -0.0009, 0.0136, ..., 0.0028, -0.0014, 0.0007],\n", - " ...,\n", - " [ 0.0039, 0.0058, -0.0080, ..., -0.0043, -0.0039, -0.0040],\n", - " [ 0.0005, -0.0047, 0.0135, ..., 0.0073, 0.0052, 0.0081],\n", - " [-0.0088, -0.0020, 0.0143, ..., 0.0133, 0.0091, 0.0072]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.15.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0296, -0.0106, 0.0387, ..., -0.0194, -0.0090, -0.0105],\n", - " [ 0.0402, 0.0177, 0.0116, ..., 0.0274, 0.0108, -0.0268],\n", - " [-0.0306, 0.0236, 0.0116, ..., -0.0145, 0.0411, 0.0181],\n", - " ...,\n", - " [-0.0032, -0.0044, -0.0173, ..., 0.0181, -0.0130, -0.0105],\n", - " [ 0.0389, 0.0156, -0.0165, ..., 0.0283, -0.0330, 0.0074],\n", - " [-0.0487, 0.0026, 0.0130, ..., -0.0018, 0.0217, -0.0254]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.15.layer.1.EncDecAttention.v.lora_B.weight': tensor([[-0.0079, 0.0204, -0.0110, ..., 0.0214, 0.0234, -0.0076],\n", - " [ 0.0076, -0.0051, 0.0005, ..., 0.0017, -0.0050, 0.0087],\n", - " [-0.0129, 0.0082, -0.0114, ..., 0.0144, 0.0113, -0.0152],\n", - " ...,\n", - " [-0.0073, 0.0115, -0.0072, ..., 0.0007, 0.0094, -0.0099],\n", - " [ 0.0005, 0.0024, -0.0036, ..., 0.0100, 0.0121, -0.0115],\n", - " [-0.0065, -0.0010, -0.0054, ..., 0.0009, 0.0019, -0.0048]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.16.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0219, -0.0087, -0.0181, ..., 0.0393, 0.0068, -0.0218],\n", - " [-0.0422, -0.0278, 0.0179, ..., 0.0154, 0.0026, 0.0087],\n", - " [ 0.0098, -0.0264, 0.0298, ..., 0.0034, -0.0046, -0.0290],\n", - " ...,\n", - " [ 0.0221, -0.0257, 0.0337, ..., 0.0161, -0.0059, -0.0238],\n", - " [-0.0034, -0.0086, -0.0082, ..., -0.0122, -0.0405, 0.0123],\n", - " [ 0.0056, -0.0267, -0.0337, ..., 0.0398, -0.0013, 0.0338]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.16.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0045, 0.0062, 0.0051, ..., -0.0066, -0.0064, 0.0069],\n", - " [-0.0070, -0.0047, -0.0063, ..., 0.0037, 0.0037, -0.0014],\n", - " [ 0.0058, 0.0074, -0.0024, ..., -0.0094, -0.0086, 0.0102],\n", - " ...,\n", - " [-0.0225, -0.0194, 0.0190, ..., 0.0209, 0.0221, -0.0232],\n", - " [-0.0137, -0.0164, 0.0105, ..., 0.0161, 0.0164, -0.0072],\n", - " [ 0.0036, 0.0069, 0.0028, ..., -0.0018, -0.0002, -0.0341]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.16.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0219, -0.0161, 0.0182, ..., 0.0006, 0.0182, 0.0124],\n", - " [ 0.0040, 0.0343, 0.0194, ..., -0.0299, -0.0435, 0.0343],\n", - " [ 0.0039, -0.0184, -0.0296, ..., -0.0075, 0.0109, -0.0545],\n", - " ...,\n", - " [ 0.0032, 0.0418, -0.0225, ..., -0.0043, -0.0383, -0.0002],\n", - " [ 0.0360, 0.0251, 0.0007, ..., -0.0507, -0.0307, 0.0645],\n", - " [ 0.0341, 0.0468, 0.0273, ..., -0.0019, -0.0207, 0.0422]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.16.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0233, -0.0240, 0.0242, ..., -0.0231, -0.0250, -0.0250],\n", - " [-0.0088, -0.0091, 0.0081, ..., -0.0089, -0.0099, -0.0091],\n", - " [ 0.0041, 0.0020, 0.0018, ..., 0.0014, 0.0007, 0.0016],\n", - " ...,\n", - " [-0.0078, -0.0184, 0.0138, ..., -0.0104, -0.0209, -0.0149],\n", - " [ 0.0161, 0.0129, -0.0111, ..., 0.0140, 0.0109, 0.0143],\n", - " [ 0.0067, 0.0030, -0.0056, ..., 0.0036, -0.0010, 0.0049]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.16.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0027, -0.0303, 0.0067, ..., -0.0229, 0.0209, -0.0409],\n", - " [-0.0099, -0.0124, 0.0144, ..., -0.0010, -0.0295, 0.0039],\n", - " [-0.0113, 0.0164, 0.0156, ..., 0.0317, 0.0263, 0.0163],\n", - " ...,\n", - " [ 0.0317, -0.0133, 0.0245, ..., -0.0112, 0.0027, 0.0285],\n", - " [ 0.0233, 0.0047, -0.0222, ..., -0.0179, 0.0321, 0.0024],\n", - " [ 0.0352, -0.0024, -0.0367, ..., 0.0203, -0.0148, -0.0224]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.16.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0052, -0.0046, 0.0103, ..., 0.0058, 0.0120, 0.0090],\n", - " [-0.0022, 0.0010, -0.0026, ..., -0.0087, -0.0056, -0.0035],\n", - " [ 0.0039, -0.0125, 0.0089, ..., -0.0214, 0.0074, 0.0066],\n", - " ...,\n", - " [ 0.0275, -0.0290, 0.0311, ..., -0.0106, 0.0308, 0.0293],\n", - " [ 0.0141, -0.0132, 0.0107, ..., -0.0075, 0.0121, 0.0140],\n", - " [-0.0304, 0.0323, -0.0312, ..., 0.0101, -0.0350, -0.0371]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.16.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0211, -0.0161, 0.0199, ..., -0.0291, 0.0098, 0.0633],\n", - " [ 0.0079, -0.0033, -0.0019, ..., 0.0192, -0.0015, -0.0375],\n", - " [-0.0076, -0.0288, 0.0126, ..., 0.0054, -0.0087, 0.0139],\n", - " ...,\n", - " [ 0.0009, -0.0255, 0.0367, ..., -0.0258, 0.0083, 0.0221],\n", - " [ 0.0277, 0.0011, 0.0027, ..., -0.0070, 0.0137, 0.0379],\n", - " [ 0.0215, 0.0060, -0.0134, ..., 0.0053, -0.0184, -0.0192]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.16.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 0.0108, 0.0078, 0.0119, ..., 0.0124, 0.0036, -0.0123],\n", - " [ 0.0202, -0.0023, 0.0220, ..., 0.0209, -0.0016, -0.0214],\n", - " [-0.0082, 0.0027, 0.0075, ..., -0.0011, -0.0155, 0.0050],\n", - " ...,\n", - " [-0.0056, 0.0035, -0.0025, ..., -0.0039, -0.0004, 0.0047],\n", - " [-0.0191, 0.0135, -0.0108, ..., -0.0092, -0.0125, 0.0128],\n", - " [ 0.0191, -0.0031, 0.0218, ..., 0.0199, 0.0088, -0.0202]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.17.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0110, 0.0033, -0.0113, ..., 0.0048, 0.0247, 0.0054],\n", - " [ 0.0127, -0.0212, -0.0472, ..., -0.0268, 0.0428, -0.0202],\n", - " [-0.0078, 0.0169, 0.0111, ..., -0.0042, -0.0003, -0.0235],\n", - " ...,\n", - " [-0.0244, -0.0122, -0.0304, ..., -0.0329, 0.0367, -0.0051],\n", - " [ 0.0205, 0.0283, 0.0346, ..., 0.0034, 0.0209, 0.0203],\n", - " [-0.0127, 0.0077, 0.0263, ..., 0.0332, -0.0028, 0.0078]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.17.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0027, 0.0025, 0.0065, ..., 0.0052, -0.0079, -0.0049],\n", - " [ 0.0232, 0.0130, 0.0136, ..., 0.0119, -0.0129, -0.0109],\n", - " [-0.0266, -0.0229, -0.0219, ..., -0.0216, 0.0204, 0.0209],\n", - " ...,\n", - " [ 0.0131, 0.0106, 0.0096, ..., 0.0094, -0.0127, -0.0143],\n", - " [ 0.0107, 0.0033, 0.0019, ..., 0.0027, -0.0010, -0.0041],\n", - " [-0.0081, -0.0046, -0.0037, ..., -0.0038, 0.0019, 0.0018]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.17.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0325, 0.0404, -0.0134, ..., 0.0062, -0.0371, 0.0013],\n", - " [-0.0041, -0.0016, 0.0144, ..., -0.0214, -0.0108, -0.0138],\n", - " [-0.0005, 0.0322, -0.0368, ..., -0.0403, 0.0035, -0.0059],\n", - " ...,\n", - " [ 0.0045, 0.0389, 0.0109, ..., -0.0075, -0.0146, 0.0066],\n", - " [-0.0320, -0.0356, 0.0048, ..., 0.0524, 0.0445, -0.0348],\n", - " [-0.0180, -0.0509, 0.0261, ..., 0.0295, 0.0356, -0.0276]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.17.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0078, 0.0004, 0.0029, ..., -0.0020, 0.0022, 0.0038],\n", - " [ 0.0122, 0.0044, 0.0141, ..., 0.0164, -0.0155, -0.0095],\n", - " [-0.0067, 0.0061, 0.0006, ..., -0.0035, 0.0040, 0.0004],\n", - " ...,\n", - " [ 0.0147, 0.0099, 0.0087, ..., 0.0133, -0.0135, -0.0156],\n", - " [ 0.0059, -0.0010, 0.0012, ..., 0.0056, -0.0035, 0.0037],\n", - " [ 0.0002, 0.0016, 0.0042, ..., 0.0002, 0.0006, 0.0034]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.17.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0348, -0.0118, 0.0138, ..., -0.0390, 0.0242, -0.0179],\n", - " [ 0.0164, -0.0063, 0.0224, ..., -0.0279, 0.0316, -0.0096],\n", - " [ 0.0018, -0.0266, 0.0202, ..., -0.0475, -0.0246, -0.0220],\n", - " ...,\n", - " [ 0.0014, -0.0097, -0.0064, ..., 0.0311, 0.0035, -0.0014],\n", - " [ 0.0395, 0.0291, 0.0073, ..., -0.0023, -0.0247, 0.0419],\n", - " [ 0.0226, 0.0164, -0.0280, ..., 0.0056, 0.0016, 0.0153]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.17.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0078, 0.0093, 0.0078, ..., -0.0093, -0.0051, -0.0021],\n", - " [-0.0284, -0.0147, -0.0206, ..., 0.0180, 0.0384, 0.0393],\n", - " [ 0.0054, 0.0074, 0.0035, ..., -0.0096, -0.0074, -0.0064],\n", - " ...,\n", - " [-0.0116, -0.0127, -0.0108, ..., 0.0129, 0.0128, 0.0102],\n", - " [ 0.0139, 0.0120, 0.0094, ..., -0.0132, -0.0130, -0.0141],\n", - " [ 0.0109, 0.0123, 0.0065, ..., -0.0120, -0.0124, -0.0094]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.17.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0262, -0.0628, 0.0287, ..., -0.0263, 0.0311, -0.0192],\n", - " [-0.0029, 0.0124, -0.0072, ..., 0.0043, -0.0331, -0.0429],\n", - " [ 0.0236, 0.0187, -0.0174, ..., 0.0047, 0.0140, 0.0135],\n", - " ...,\n", - " [ 0.0152, 0.0109, 0.0131, ..., -0.0012, -0.0165, 0.0063],\n", - " [-0.0361, -0.0315, 0.0343, ..., -0.0198, 0.0440, -0.0010],\n", - " [ 0.0114, 0.0118, -0.0288, ..., 0.0051, -0.0095, -0.0169]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.17.layer.1.EncDecAttention.v.lora_B.weight': tensor([[-0.0318, 0.0044, 0.0069, ..., 0.0107, -0.0189, 0.0074],\n", - " [-0.0332, 0.0219, -0.0159, ..., -0.0156, -0.0224, 0.0240],\n", - " [ 0.0288, -0.0129, 0.0086, ..., 0.0045, 0.0240, -0.0148],\n", - " ...,\n", - " [ 0.0157, 0.0096, -0.0117, ..., -0.0127, -0.0148, -0.0194],\n", - " [ 0.0155, 0.0098, -0.0099, ..., -0.0118, -0.0114, -0.0147],\n", - " [-0.0018, -0.0110, 0.0131, ..., 0.0123, 0.0138, 0.0039]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.18.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0352, -0.0196, 0.0188, ..., 0.0375, -0.0202, -0.0196],\n", - " [ 0.0414, 0.0130, -0.0185, ..., 0.0279, -0.0007, -0.0084],\n", - " [ 0.0244, -0.0149, -0.0413, ..., 0.0338, -0.0023, 0.0292],\n", - " ...,\n", - " [ 0.0105, 0.0305, 0.0255, ..., -0.0105, -0.0306, -0.0135],\n", - " [ 0.0201, 0.0205, 0.0201, ..., 0.0055, -0.0065, -0.0440],\n", - " [ 0.0362, -0.0236, 0.0030, ..., -0.0126, 0.0508, 0.0348]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.18.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0081, 0.0077, 0.0071, ..., -0.0068, 0.0024, 0.0021],\n", - " [-0.0012, 0.0028, 0.0023, ..., -0.0028, -0.0080, 0.0022],\n", - " [ 0.0074, 0.0088, 0.0088, ..., -0.0086, 0.0027, 0.0098],\n", - " ...,\n", - " [-0.0137, -0.0088, -0.0094, ..., 0.0106, 0.0117, -0.0056],\n", - " [ 0.0091, 0.0049, 0.0071, ..., -0.0084, 0.0008, -0.0027],\n", - " [-0.0011, -0.0091, -0.0071, ..., 0.0047, -0.0046, -0.0082]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.18.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0202, 0.0472, -0.0144, ..., -0.0219, 0.0103, 0.0043],\n", - " [ 0.0089, 0.0365, 0.0118, ..., -0.0073, 0.0052, 0.0454],\n", - " [-0.0188, 0.0213, -0.0220, ..., -0.0063, -0.0301, 0.0115],\n", - " ...,\n", - " [ 0.0001, 0.0202, 0.0150, ..., -0.0270, 0.0132, 0.0644],\n", - " [-0.0128, 0.0170, 0.0343, ..., 0.0307, 0.0125, 0.0154],\n", - " [ 0.0368, -0.0510, -0.0221, ..., 0.0050, 0.0360, -0.0401]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.18.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0183, 0.0209, 0.0223, ..., 0.0224, -0.0276, -0.0238],\n", - " [ 0.0018, 0.0003, -0.0033, ..., -0.0021, 0.0007, 0.0008],\n", - " [-0.0094, -0.0069, -0.0029, ..., -0.0034, 0.0021, 0.0041],\n", - " ...,\n", - " [ 0.0106, 0.0082, -0.0028, ..., -0.0003, 0.0051, 0.0002],\n", - " [ 0.0205, 0.0214, 0.0110, ..., 0.0128, -0.0095, -0.0170],\n", - " [-0.0337, -0.0304, -0.0236, ..., -0.0311, 0.0311, 0.0223]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.18.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0321, -0.0146, 0.0019, ..., 0.0236, 0.0183, -0.0471],\n", - " [-0.0162, -0.0074, -0.0296, ..., 0.0163, -0.0134, -0.0315],\n", - " [ 0.0442, -0.0036, -0.0010, ..., -0.0262, 0.0005, 0.0129],\n", - " ...,\n", - " [ 0.0039, 0.0022, -0.0167, ..., -0.0317, 0.0070, 0.0184],\n", - " [-0.0482, -0.0197, -0.0316, ..., -0.0014, -0.0049, -0.0271],\n", - " [ 0.0432, -0.0289, 0.0295, ..., -0.0390, -0.0308, 0.0124]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.18.layer.1.EncDecAttention.q.lora_B.weight': tensor([[-0.0075, -0.0108, 0.0096, ..., 0.0016, -0.0095, 0.0080],\n", - " [-0.0140, -0.0120, 0.0073, ..., 0.0127, -0.0119, 0.0123],\n", - " [-0.0056, -0.0077, 0.0043, ..., -0.0065, -0.0042, 0.0043],\n", - " ...,\n", - " [-0.0019, 0.0033, 0.0096, ..., -0.0167, -0.0077, 0.0028],\n", - " [-0.0132, -0.0107, 0.0170, ..., 0.0081, -0.0119, 0.0058],\n", - " [-0.0042, -0.0003, 0.0090, ..., 0.0026, -0.0019, 0.0004]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.18.layer.1.EncDecAttention.v.lora_A.weight': tensor([[ 0.0196, 0.0030, 0.0224, ..., -0.0038, 0.0045, 0.0215],\n", - " [ 0.0100, 0.0074, 0.0359, ..., 0.0210, 0.0242, 0.0113],\n", - " [ 0.0026, -0.0086, 0.0127, ..., 0.0132, -0.0101, 0.0025],\n", - " ...,\n", - " [ 0.0285, -0.0291, 0.0067, ..., -0.0070, -0.0483, -0.0456],\n", - " [-0.0019, -0.0021, -0.0288, ..., -0.0177, 0.0375, 0.0036],\n", - " [-0.0219, 0.0325, 0.0195, ..., -0.0087, -0.0078, 0.0244]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.18.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 0.0184, -0.0062, -0.0195, ..., -0.0182, 0.0185, 0.0185],\n", - " [ 0.0150, -0.0073, -0.0114, ..., -0.0022, 0.0037, 0.0024],\n", - " [ 0.0261, -0.0178, -0.0264, ..., -0.0250, 0.0274, 0.0250],\n", - " ...,\n", - " [ 0.0242, -0.0040, -0.0231, ..., -0.0215, 0.0186, 0.0224],\n", - " [-0.0210, 0.0187, 0.0201, ..., 0.0210, -0.0218, -0.0205],\n", - " [-0.0242, -0.0048, 0.0228, ..., 0.0187, -0.0221, -0.0207]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.19.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0391, -0.0192, 0.0063, ..., 0.0308, 0.0023, 0.0011],\n", - " [-0.0155, 0.0229, -0.0224, ..., -0.0024, 0.0059, -0.0193],\n", - " [ 0.0280, -0.0313, -0.0107, ..., -0.0039, 0.0153, 0.0012],\n", - " ...,\n", - " [ 0.0262, 0.0110, -0.0075, ..., -0.0037, 0.0194, -0.0427],\n", - " [-0.0242, -0.0382, 0.0120, ..., 0.0300, -0.0108, 0.0073],\n", - " [-0.0040, -0.0096, 0.0203, ..., -0.0097, -0.0198, -0.0229]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.19.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 6.1453e-03, 6.1973e-03, -6.8584e-03, ..., -4.5252e-03,\n", - " 6.3516e-03, 6.2948e-03],\n", - " [-4.5412e-03, -4.4871e-03, 4.0846e-03, ..., 5.9160e-03,\n", - " -4.4922e-03, -4.3107e-03],\n", - " [-6.2454e-03, -6.1674e-03, 6.7055e-03, ..., 4.5486e-03,\n", - " -6.3253e-03, -6.0943e-03],\n", - " ...,\n", - " [-6.4837e-03, -7.3207e-03, 8.4997e-03, ..., -6.3963e-05,\n", - " -3.4027e-04, 1.2465e-02],\n", - " [ 1.0956e-02, 9.6466e-03, -7.7077e-03, ..., -9.7397e-03,\n", - " 1.1114e-02, -2.4019e-02],\n", - " [ 2.1932e-02, 1.3618e-02, -1.6683e-02, ..., -1.7114e-02,\n", - " 1.7331e-02, -2.1152e-02]], device='cuda:0'), 'base_model.model.decoder.block.19.layer.0.SelfAttention.v.lora_A.weight': tensor([[-1.2191e-03, -1.8192e-02, 2.0656e-02, ..., -3.0691e-02,\n", - " -3.7022e-02, 6.2503e-02],\n", - " [-8.9603e-05, -2.4246e-02, -1.6662e-02, ..., -4.4149e-02,\n", - " -7.4380e-03, 1.4131e-02],\n", - " [-1.2862e-03, 2.0751e-02, 1.4463e-03, ..., 1.3739e-02,\n", - " 3.9745e-02, -2.3024e-02],\n", - " ...,\n", - " [ 1.6170e-03, 2.8526e-02, -1.6437e-02, ..., -4.2300e-02,\n", - " -1.4941e-02, 4.1151e-02],\n", - " [-1.9709e-02, -1.1424e-02, 1.7977e-02, ..., 3.2824e-02,\n", - " 6.1655e-03, -5.5113e-02],\n", - " [-4.6231e-03, -1.9727e-03, 2.2364e-02, ..., -3.3645e-03,\n", - " -1.6693e-02, -9.5787e-04]], device='cuda:0'), 'base_model.model.decoder.block.19.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0227, 0.0295, -0.0247, ..., 0.0254, -0.0258, 0.0219],\n", - " [ 0.0271, 0.0284, -0.0212, ..., 0.0271, -0.0269, 0.0150],\n", - " [ 0.0234, 0.0235, -0.0208, ..., 0.0246, -0.0221, 0.0152],\n", - " ...,\n", - " [ 0.0078, 0.0068, -0.0066, ..., 0.0072, -0.0073, 0.0119],\n", - " [ 0.0140, 0.0122, -0.0158, ..., 0.0121, -0.0126, 0.0089],\n", - " [-0.0119, -0.0147, 0.0114, ..., -0.0144, 0.0136, -0.0122]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.19.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0183, 0.0285, -0.0087, ..., 0.0389, -0.0304, -0.0211],\n", - " [-0.0469, 0.0042, -0.0137, ..., 0.0184, -0.0258, 0.0023],\n", - " [ 0.0009, -0.0048, -0.0200, ..., 0.0073, -0.0202, 0.0201],\n", - " ...,\n", - " [-0.0237, 0.0142, 0.0049, ..., 0.0029, -0.0308, 0.0134],\n", - " [-0.0482, 0.0319, -0.0207, ..., 0.0067, -0.0235, -0.0195],\n", - " [ 0.0306, -0.0269, 0.0360, ..., -0.0022, -0.0164, 0.0399]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.19.layer.1.EncDecAttention.q.lora_B.weight': tensor([[-0.0137, -0.0193, 0.0165, ..., -0.0196, -0.0190, 0.0197],\n", - " [ 0.0025, -0.0017, 0.0025, ..., -0.0013, -0.0026, 0.0021],\n", - " [-0.0140, -0.0209, 0.0177, ..., -0.0156, -0.0172, 0.0192],\n", - " ...,\n", - " [ 0.0085, 0.0040, -0.0062, ..., 0.0022, 0.0034, -0.0028],\n", - " [-0.0045, -0.0048, 0.0089, ..., -0.0104, -0.0076, 0.0103],\n", - " [-0.0155, -0.0158, 0.0103, ..., -0.0147, -0.0128, 0.0156]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.19.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0591, -0.0330, 0.0479, ..., -0.0135, 0.0269, -0.0320],\n", - " [ 0.0116, 0.0332, 0.0454, ..., -0.0533, 0.0502, 0.0280],\n", - " [ 0.0680, 0.0278, -0.0021, ..., 0.0058, -0.0372, 0.0176],\n", - " ...,\n", - " [-0.0592, -0.0317, 0.0202, ..., 0.0044, 0.0342, -0.0396],\n", - " [-0.0679, -0.0569, 0.0012, ..., 0.0163, 0.0423, -0.0169],\n", - " [ 0.0339, 0.0452, -0.0357, ..., 0.0423, -0.0517, 0.0218]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.19.layer.1.EncDecAttention.v.lora_B.weight': tensor([[-0.0204, -0.0194, -0.0167, ..., -0.0192, 0.0034, 0.0193],\n", - " [-0.0046, -0.0046, -0.0007, ..., -0.0025, 0.0107, 0.0048],\n", - " [-0.0109, -0.0102, -0.0077, ..., -0.0106, 0.0037, 0.0111],\n", - " ...,\n", - " [ 0.0105, -0.0051, -0.0013, ..., -0.0022, -0.0002, 0.0012],\n", - " [ 0.0298, 0.0384, -0.0381, ..., 0.0380, 0.0370, -0.0360],\n", - " [-0.0161, -0.0247, 0.0242, ..., -0.0196, -0.0242, 0.0225]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.20.layer.0.SelfAttention.q.lora_A.weight': tensor([[-0.0330, -0.0107, 0.0211, ..., 0.0107, 0.0013, -0.0105],\n", - " [ 0.0142, -0.0134, 0.0259, ..., 0.0185, 0.0278, 0.0195],\n", - " [-0.0087, -0.0145, -0.0396, ..., 0.0157, -0.0085, -0.0097],\n", - " ...,\n", - " [-0.0029, 0.0066, -0.0174, ..., 0.0006, -0.0260, -0.0203],\n", - " [ 0.0232, 0.0241, 0.0129, ..., -0.0420, 0.0071, -0.0258],\n", - " [ 0.0318, 0.0162, 0.0062, ..., 0.0249, 0.0159, -0.0238]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.20.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 0.0066, 0.0075, 0.0076, ..., -0.0074, -0.0073, -0.0012],\n", - " [-0.0060, -0.0079, -0.0063, ..., 0.0062, 0.0067, 0.0021],\n", - " [-0.0026, -0.0072, -0.0017, ..., 0.0012, 0.0013, -0.0047],\n", - " ...,\n", - " [-0.0108, -0.0189, -0.0058, ..., 0.0137, 0.0104, 0.0008],\n", - " [-0.0061, -0.0086, -0.0066, ..., 0.0030, 0.0012, -0.0078],\n", - " [ 0.0148, 0.0042, 0.0149, ..., -0.0157, -0.0143, -0.0109]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.20.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0134, 0.0197, 0.0471, ..., -0.0059, -0.0069, 0.0288],\n", - " [-0.0282, -0.0023, -0.0330, ..., 0.0061, 0.0268, -0.0604],\n", - " [-0.0080, 0.0056, 0.0321, ..., -0.0319, 0.0065, 0.0143],\n", - " ...,\n", - " [-0.0269, 0.0138, 0.0644, ..., 0.0223, -0.0018, 0.0240],\n", - " [-0.0010, -0.0047, 0.0269, ..., 0.0136, 0.0345, -0.0001],\n", - " [-0.0323, 0.0287, 0.0055, ..., 0.0195, 0.0189, -0.0179]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.20.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0061, 0.0025, -0.0070, ..., 0.0109, 0.0056, 0.0067],\n", - " [-0.0246, 0.0245, -0.0271, ..., -0.0096, 0.0270, 0.0278],\n", - " [ 0.0282, -0.0249, 0.0236, ..., 0.0410, -0.0145, -0.0263],\n", - " ...,\n", - " [-0.0128, 0.0086, 0.0037, ..., -0.0332, -0.0036, 0.0066],\n", - " [-0.0181, 0.0180, -0.0174, ..., -0.0017, 0.0149, 0.0184],\n", - " [-0.0168, 0.0144, -0.0048, ..., 0.0045, 0.0113, 0.0176]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.20.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0309, -0.0433, -0.0072, ..., 0.0526, -0.0267, 0.0111],\n", - " [ 0.0322, 0.0130, -0.0075, ..., -0.0538, 0.0289, 0.0092],\n", - " [ 0.0119, 0.0142, 0.0159, ..., -0.0186, 0.0124, 0.0052],\n", - " ...,\n", - " [-0.0182, -0.0393, 0.0251, ..., 0.0051, -0.0294, 0.0166],\n", - " [ 0.0204, -0.0257, 0.0171, ..., 0.0160, 0.0026, 0.0258],\n", - " [-0.0115, -0.0061, 0.0026, ..., -0.0053, 0.0131, 0.0137]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.20.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0007, -0.0135, -0.0022, ..., -0.0014, 0.0102, -0.0091],\n", - " [-0.0013, -0.0089, -0.0008, ..., -0.0011, 0.0063, -0.0077],\n", - " [ 0.0135, -0.0018, -0.0098, ..., 0.0087, 0.0182, -0.0043],\n", - " ...,\n", - " [ 0.0196, -0.0158, -0.0186, ..., 0.0112, 0.0094, -0.0163],\n", - " [ 0.0239, -0.0141, -0.0202, ..., 0.0215, 0.0194, -0.0229],\n", - " [-0.0105, 0.0090, 0.0092, ..., -0.0076, -0.0030, 0.0103]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.20.layer.1.EncDecAttention.v.lora_A.weight': tensor([[ 0.0300, -0.0317, -0.0016, ..., 0.0477, -0.0341, -0.0443],\n", - " [ 0.0130, -0.0124, 0.0146, ..., 0.0177, 0.0036, 0.0013],\n", - " [-0.0037, -0.0089, -0.0270, ..., -0.0676, 0.0243, 0.0449],\n", - " ...,\n", - " [ 0.0045, 0.0037, -0.0003, ..., -0.0588, 0.0432, 0.0416],\n", - " [ 0.0265, -0.0409, -0.0213, ..., 0.0781, -0.0029, -0.0608],\n", - " [-0.0193, -0.0034, 0.0216, ..., -0.0217, 0.0456, 0.0479]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.20.layer.1.EncDecAttention.v.lora_B.weight': tensor([[-0.0056, -0.0064, 0.0067, ..., -0.0014, -0.0009, 0.0006],\n", - " [-0.0254, -0.0245, 0.0249, ..., 0.0180, -0.0263, 0.0243],\n", - " [-0.0236, -0.0116, 0.0207, ..., 0.0163, -0.0246, 0.0150],\n", - " ...,\n", - " [ 0.0120, 0.0035, -0.0083, ..., -0.0079, 0.0130, -0.0055],\n", - " [ 0.0174, 0.0187, -0.0193, ..., -0.0140, 0.0158, -0.0161],\n", - " [ 0.0092, 0.0099, -0.0160, ..., -0.0007, 0.0068, -0.0113]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.21.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0343, 0.0204, 0.0275, ..., 0.0053, -0.0517, -0.0413],\n", - " [-0.0248, -0.0211, 0.0076, ..., 0.0244, 0.0250, 0.0150],\n", - " [-0.0068, -0.0234, 0.0406, ..., 0.0005, -0.0413, -0.0170],\n", - " ...,\n", - " [ 0.0428, -0.0168, -0.0036, ..., -0.0061, -0.0196, 0.0055],\n", - " [-0.0289, 0.0417, -0.0210, ..., 0.0379, 0.0039, 0.0105],\n", - " [ 0.0070, 0.0048, 0.0043, ..., -0.0240, -0.0300, -0.0242]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.21.layer.0.SelfAttention.q.lora_B.weight': tensor([[ 4.4348e-03, -2.6485e-03, 2.1192e-03, ..., 6.5235e-04,\n", - " -4.2449e-03, 2.7613e-03],\n", - " [ 1.5684e-03, -2.8341e-04, 4.5689e-05, ..., -1.3167e-03,\n", - " -6.2685e-04, -4.6340e-04],\n", - " [ 5.4143e-04, 6.6562e-04, -9.4752e-04, ..., -2.5828e-03,\n", - " 7.1928e-04, -1.4943e-03],\n", - " ...,\n", - " [ 3.1052e-03, -1.0300e-02, 9.1300e-04, ..., 5.9527e-03,\n", - " 1.4728e-03, 2.2390e-03],\n", - " [-1.0343e-02, 3.8973e-03, -1.0227e-02, ..., -2.1103e-03,\n", - " 1.1451e-02, -1.0909e-02],\n", - " [ 9.7546e-03, -4.5729e-03, 9.8230e-03, ..., 6.4971e-03,\n", - " -9.7390e-03, 9.7719e-03]], device='cuda:0'), 'base_model.model.decoder.block.21.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0403, -0.0040, -0.0013, ..., -0.0024, 0.0256, 0.0428],\n", - " [-0.0482, 0.0095, -0.0140, ..., -0.0013, 0.0166, -0.0315],\n", - " [ 0.0045, 0.0172, 0.0354, ..., -0.0425, -0.0294, 0.0075],\n", - " ...,\n", - " [-0.0317, -0.0092, -0.0335, ..., 0.0219, 0.0336, -0.0158],\n", - " [-0.0078, -0.0031, -0.0020, ..., -0.0177, -0.0036, -0.0421],\n", - " [ 0.0143, -0.0200, -0.0190, ..., 0.0009, -0.0299, 0.0199]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.21.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0263, -0.0255, 0.0235, ..., -0.0238, -0.0280, 0.0286],\n", - " [ 0.0201, -0.0229, 0.0209, ..., -0.0200, -0.0184, 0.0127],\n", - " [-0.0039, 0.0015, -0.0030, ..., -0.0007, 0.0009, -0.0090],\n", - " ...,\n", - " [ 0.0110, -0.0022, 0.0084, ..., -0.0041, 0.0051, 0.0059],\n", - " [ 0.0110, -0.0111, 0.0048, ..., -0.0118, -0.0143, 0.0013],\n", - " [ 0.0178, -0.0232, 0.0145, ..., -0.0203, -0.0183, 0.0250]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.21.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0286, -0.0409, -0.0219, ..., 0.0076, 0.0002, -0.0041],\n", - " [-0.0026, 0.0309, 0.0021, ..., -0.0009, -0.0211, 0.0268],\n", - " [ 0.0109, 0.0248, -0.0301, ..., 0.0485, -0.0336, -0.0063],\n", - " ...,\n", - " [-0.0150, -0.0146, -0.0259, ..., -0.0196, 0.0254, -0.0009],\n", - " [ 0.0092, -0.0122, -0.0444, ..., 0.0018, -0.0069, 0.0313],\n", - " [-0.0075, -0.0570, -0.0182, ..., 0.0037, 0.0248, -0.0176]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.21.layer.1.EncDecAttention.q.lora_B.weight': tensor([[-0.0168, 0.0255, 0.0151, ..., -0.0183, -0.0177, -0.0194],\n", - " [-0.0206, 0.0197, 0.0183, ..., -0.0195, -0.0225, -0.0275],\n", - " [ 0.0361, -0.0352, -0.0348, ..., 0.0332, 0.0327, 0.0342],\n", - " ...,\n", - " [ 0.0194, -0.0234, -0.0214, ..., 0.0162, 0.0178, 0.0205],\n", - " [-0.0017, 0.0059, 0.0040, ..., -0.0048, -0.0056, -0.0058],\n", - " [ 0.0077, -0.0114, -0.0011, ..., 0.0101, 0.0192, 0.0077]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.21.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-3.5695e-02, 1.6790e-02, 5.7587e-04, ..., -1.6495e-02,\n", - " 1.2606e-02, 4.8337e-02],\n", - " [ 2.3041e-02, -3.3336e-02, -2.2698e-02, ..., 5.9429e-02,\n", - " -3.1289e-02, -4.4992e-02],\n", - " [-1.5568e-02, -5.2893e-02, -1.2566e-02, ..., 4.0882e-02,\n", - " 1.0086e-02, -9.9746e-03],\n", - " ...,\n", - " [ 2.4377e-02, -3.7222e-02, 1.3223e-02, ..., 5.4595e-02,\n", - " -3.3970e-02, -1.3829e-03],\n", - " [ 1.6691e-02, 5.6453e-02, -1.8042e-02, ..., -7.6591e-02,\n", - " 9.8997e-05, 4.6439e-02],\n", - " [-9.6895e-03, -2.9741e-02, 1.2167e-02, ..., 2.4567e-02,\n", - " 9.3858e-03, -3.2569e-02]], device='cuda:0'), 'base_model.model.decoder.block.21.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 0.0311, -0.0313, -0.0311, ..., -0.0332, 0.0247, -0.0304],\n", - " [ 0.0029, 0.0026, 0.0029, ..., 0.0035, -0.0013, -0.0060],\n", - " [-0.0258, 0.0229, 0.0264, ..., 0.0237, -0.0149, 0.0358],\n", - " ...,\n", - " [ 0.0278, -0.0260, -0.0260, ..., -0.0240, 0.0291, -0.0316],\n", - " [ 0.0034, 0.0013, -0.0023, ..., -0.0007, -0.0021, -0.0052],\n", - " [ 0.0103, -0.0093, -0.0117, ..., -0.0091, 0.0129, -0.0151]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.22.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0075, 0.0101, 0.0066, ..., 0.0137, -0.0290, -0.0332],\n", - " [-0.0080, -0.0053, -0.0135, ..., 0.0092, -0.0176, -0.0109],\n", - " [-0.0237, -0.0107, 0.0388, ..., -0.0160, -0.0025, -0.0372],\n", - " ...,\n", - " [ 0.0404, -0.0182, 0.0073, ..., 0.0007, 0.0072, -0.0113],\n", - " [ 0.0124, -0.0111, 0.0153, ..., -0.0410, -0.0116, -0.0214],\n", - " [-0.0053, -0.0287, -0.0165, ..., -0.0356, 0.0135, -0.0157]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.22.layer.0.SelfAttention.q.lora_B.weight': tensor([[-0.0063, 0.0048, 0.0027, ..., 0.0076, -0.0064, -0.0074],\n", - " [ 0.0014, -0.0117, -0.0015, ..., -0.0004, 0.0117, 0.0018],\n", - " [ 0.0156, -0.0110, 0.0123, ..., -0.0135, 0.0156, 0.0141],\n", - " ...,\n", - " [ 0.0047, -0.0093, 0.0169, ..., -0.0119, 0.0120, 0.0156],\n", - " [-0.0021, 0.0090, -0.0040, ..., 0.0050, -0.0092, -0.0039],\n", - " [-0.0010, -0.0057, -0.0066, ..., -0.0030, 0.0066, -0.0035]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.22.layer.0.SelfAttention.v.lora_A.weight': tensor([[ 0.0456, -0.0016, -0.0174, ..., 0.0192, -0.0071, -0.0623],\n", - " [ 0.0340, 0.0215, -0.0414, ..., -0.0354, -0.0016, -0.0324],\n", - " [ 0.0472, -0.0280, -0.0585, ..., -0.0303, -0.0207, -0.0495],\n", - " ...,\n", - " [-0.0485, -0.0319, -0.0091, ..., 0.0348, 0.0049, 0.0335],\n", - " [-0.0007, 0.0192, -0.0118, ..., -0.0202, -0.0005, -0.0519],\n", - " [-0.0050, -0.0187, 0.0160, ..., 0.0365, 0.0291, 0.0377]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.22.layer.0.SelfAttention.v.lora_B.weight': tensor([[-0.0233, -0.0288, -0.0115, ..., 0.0231, -0.0130, 0.0227],\n", - " [-0.0201, -0.0285, -0.0253, ..., 0.0275, -0.0260, 0.0299],\n", - " [-0.0058, -0.0212, -0.0120, ..., 0.0103, -0.0118, 0.0128],\n", - " ...,\n", - " [ 0.0127, 0.0124, 0.0190, ..., -0.0125, 0.0157, -0.0204],\n", - " [-0.0180, -0.0212, -0.0175, ..., 0.0185, -0.0135, 0.0195],\n", - " [-0.0200, -0.0104, -0.0042, ..., 0.0166, -0.0149, 0.0107]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.22.layer.1.EncDecAttention.q.lora_A.weight': tensor([[-0.0084, 0.0150, -0.0145, ..., -0.0232, -0.0257, -0.0149],\n", - " [ 0.0048, -0.0269, 0.0020, ..., -0.0380, -0.0280, 0.0107],\n", - " [-0.0511, 0.0117, -0.0309, ..., -0.0178, -0.0143, 0.0124],\n", - " ...,\n", - " [ 0.0064, 0.0060, 0.0122, ..., 0.0043, -0.0113, 0.0389],\n", - " [-0.0138, 0.0269, 0.0226, ..., 0.0071, -0.0021, -0.0120],\n", - " [-0.0171, 0.0366, -0.0063, ..., 0.0260, 0.0141, -0.0228]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.22.layer.1.EncDecAttention.q.lora_B.weight': tensor([[ 0.0172, 0.0150, -0.0139, ..., -0.0159, -0.0151, -0.0160],\n", - " [ 0.0052, 0.0058, 0.0026, ..., -0.0062, -0.0036, -0.0068],\n", - " [ 0.0252, 0.0213, -0.0235, ..., -0.0217, -0.0223, -0.0177],\n", - " ...,\n", - " [-0.0061, -0.0087, 0.0110, ..., 0.0117, 0.0126, 0.0129],\n", - " [-0.0169, -0.0116, 0.0146, ..., 0.0165, 0.0159, 0.0200],\n", - " [ 0.0153, 0.0157, -0.0118, ..., -0.0148, -0.0138, -0.0143]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.22.layer.1.EncDecAttention.v.lora_A.weight': tensor([[ 0.0323, 0.0615, -0.0286, ..., -0.0212, 0.0202, 0.0314],\n", - " [-0.0259, 0.0106, -0.0415, ..., -0.0654, -0.0317, 0.0154],\n", - " [-0.0052, -0.0413, 0.0448, ..., -0.0346, 0.0133, -0.0400],\n", - " ...,\n", - " [ 0.0140, 0.0323, -0.0152, ..., -0.0569, -0.0013, 0.0601],\n", - " [-0.0142, -0.0644, 0.0147, ..., 0.0293, -0.0159, -0.0258],\n", - " [ 0.0235, 0.0580, -0.0371, ..., -0.0377, -0.0346, 0.0627]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.22.layer.1.EncDecAttention.v.lora_B.weight': tensor([[ 0.0026, 0.0069, 0.0095, ..., -0.0012, -0.0045, -0.0018],\n", - " [ 0.0145, 0.0194, 0.0195, ..., 0.0182, -0.0195, 0.0177],\n", - " [ 0.0069, 0.0107, 0.0172, ..., 0.0084, -0.0127, 0.0161],\n", - " ...,\n", - " [ 0.0442, 0.0403, -0.0388, ..., 0.0492, -0.0602, 0.0562],\n", - " [-0.0228, -0.0222, 0.0388, ..., -0.0255, 0.0293, -0.0299],\n", - " [-0.0342, -0.0308, 0.0272, ..., -0.0349, 0.0436, -0.0529]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.23.layer.0.SelfAttention.q.lora_A.weight': tensor([[ 0.0229, 0.0198, -0.0208, ..., -0.0067, 0.0257, -0.0064],\n", - " [-0.0030, -0.0127, 0.0058, ..., -0.0117, 0.0089, 0.0005],\n", - " [ 0.0391, 0.0234, -0.0144, ..., 0.0433, 0.0134, 0.0189],\n", - " ...,\n", - " [-0.0255, -0.0331, 0.0175, ..., -0.0176, -0.0498, -0.0518],\n", - " [ 0.0129, 0.0078, -0.0142, ..., -0.0030, 0.0345, -0.0058],\n", - " [-0.0361, -0.0408, 0.0383, ..., -0.0299, -0.0016, -0.0264]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.23.layer.0.SelfAttention.q.lora_B.weight': tensor([[-9.2302e-03, 4.7082e-04, -1.6876e-03, ..., 1.5358e-03,\n", - " 3.6666e-04, 2.2397e-03],\n", - " [ 6.2450e-03, -8.7633e-05, 1.2780e-03, ..., -2.2387e-03,\n", - " -7.6455e-03, -3.6240e-03],\n", - " [ 7.5522e-03, 2.7108e-03, 4.8920e-03, ..., -3.7219e-03,\n", - " 1.2247e-03, -5.4539e-03],\n", - " ...,\n", - " [ 8.3448e-03, 3.1519e-03, 2.2037e-03, ..., 1.5882e-03,\n", - " 2.2505e-03, -1.0635e-03],\n", - " [ 3.2382e-03, 8.2808e-03, 8.4877e-03, ..., -5.4786e-03,\n", - " 9.0481e-03, -9.2424e-03],\n", - " [-3.7960e-03, -1.2958e-02, -1.1950e-02, ..., 9.1770e-03,\n", - " -1.1545e-02, 7.7533e-03]], device='cuda:0'), 'base_model.model.decoder.block.23.layer.0.SelfAttention.v.lora_A.weight': tensor([[-0.0188, 0.0154, 0.0271, ..., -0.0192, 0.0154, 0.0396],\n", - " [ 0.0480, 0.0190, -0.0210, ..., 0.0008, 0.0105, -0.0560],\n", - " [-0.0263, -0.0329, 0.0212, ..., -0.0381, 0.0282, 0.0525],\n", - " ...,\n", - " [-0.0505, 0.0039, 0.0447, ..., -0.0066, -0.0126, 0.0477],\n", - " [-0.0024, -0.0017, 0.0043, ..., -0.0188, -0.0137, 0.0257],\n", - " [-0.0486, 0.0265, 0.0539, ..., -0.0064, 0.0338, 0.0425]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.23.layer.0.SelfAttention.v.lora_B.weight': tensor([[ 0.0036, -0.0072, 0.0104, ..., 0.0092, 0.0054, 0.0088],\n", - " [ 0.0053, -0.0083, 0.0139, ..., 0.0087, 0.0064, 0.0107],\n", - " [-0.0054, 0.0058, -0.0056, ..., -0.0039, -0.0077, -0.0046],\n", - " ...,\n", - " [ 0.0016, -0.0062, 0.0073, ..., 0.0109, 0.0018, 0.0105],\n", - " [-0.0155, 0.0177, -0.0152, ..., -0.0161, -0.0173, -0.0173],\n", - " [ 0.0070, -0.0056, -0.0022, ..., 0.0054, 0.0060, -0.0005]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.23.layer.1.EncDecAttention.q.lora_A.weight': tensor([[ 0.0013, -0.0205, -0.0237, ..., -0.0034, 0.0223, -0.0525],\n", - " [-0.0195, -0.0254, -0.0189, ..., 0.0134, 0.0543, -0.0611],\n", - " [ 0.0078, 0.0246, -0.0145, ..., 0.0244, 0.0562, -0.0179],\n", - " ...,\n", - " [-0.0164, -0.0022, -0.0291, ..., -0.0274, -0.0185, -0.0195],\n", - " [ 0.0101, 0.0265, 0.0126, ..., 0.0248, 0.0240, -0.0185],\n", - " [-0.0226, -0.0008, -0.0094, ..., 0.0020, -0.0359, -0.0004]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.23.layer.1.EncDecAttention.q.lora_B.weight': tensor([[-1.3781e-02, -1.4603e-02, -1.4217e-02, ..., -9.6741e-03,\n", - " -1.7460e-02, 1.1197e-02],\n", - " [ 2.7672e-03, 5.4598e-03, -8.9388e-05, ..., 2.6880e-03,\n", - " 1.1293e-02, -9.0429e-03],\n", - " [ 3.1708e-03, -5.6693e-04, -4.4513e-03, ..., 1.0293e-03,\n", - " 1.9967e-03, -4.6050e-03],\n", - " ...,\n", - " [-1.9154e-02, 2.7257e-03, -2.3589e-03, ..., 1.8899e-03,\n", - " 6.3871e-03, -4.6289e-03],\n", - " [ 1.3242e-03, 1.7306e-03, -3.5451e-03, ..., -3.5201e-03,\n", - " -5.5426e-03, 4.4722e-03],\n", - " [ 1.3347e-02, -6.4381e-03, -1.2333e-02, ..., -5.1522e-03,\n", - " -1.4272e-02, 1.4489e-02]], device='cuda:0'), 'base_model.model.decoder.block.23.layer.1.EncDecAttention.v.lora_A.weight': tensor([[-0.0109, -0.0416, 0.0268, ..., 0.0437, -0.0096, -0.0408],\n", - " [-0.0006, -0.0368, -0.0175, ..., 0.0711, -0.0297, -0.0360],\n", - " [-0.0097, -0.0302, -0.0101, ..., 0.0609, -0.0160, -0.0315],\n", - " ...,\n", - " [-0.0348, 0.0250, 0.0112, ..., -0.0480, 0.0094, 0.0005],\n", - " [-0.0217, -0.0358, -0.0137, ..., 0.0188, -0.0225, -0.0686],\n", - " [-0.0249, -0.0334, -0.0092, ..., 0.0849, -0.0285, -0.0345]],\n", - " device='cuda:0'), 'base_model.model.decoder.block.23.layer.1.EncDecAttention.v.lora_B.weight': tensor([[-0.0244, -0.0160, -0.0185, ..., -0.0134, -0.0150, -0.0029],\n", - " [-0.0028, 0.0118, 0.0127, ..., 0.0023, 0.0139, 0.0067],\n", - " [ 0.0218, 0.0264, 0.0261, ..., 0.0093, 0.0255, 0.0199],\n", - " ...,\n", - " [-0.0049, -0.0083, -0.0071, ..., -0.0327, -0.0097, 0.0223],\n", - " [ 0.0283, 0.0257, 0.0250, ..., 0.0135, 0.0239, -0.0075],\n", - " [ 0.0004, -0.0003, -0.0002, ..., -0.0289, -0.0049, 0.0312]],\n", - " device='cuda:0')}\n" - ] - } - ], + "outputs": [], "source": [ "# saving model\n", "state_dict = get_pet_model_state_dict(model)\n", diff --git a/examples/pet_prefix_tuning_seq2seq.ipynb b/examples/pet_prefix_tuning_seq2seq.ipynb index a1d5dc4..b574aff 100644 --- a/examples/pet_prefix_tuning_seq2seq.ipynb +++ b/examples/pet_prefix_tuning_seq2seq.ipynb @@ -40,1473 +40,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "8d0850ac", "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "3d73e8a9f76d4a2e918238f8ad0b2a30", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Downloading: 0%| | 0.00/1.20k [00:00