wassname/peft
1
0
Fork 0
mirror of https://github.com/wassname/peft.git synced 2026-09-26 14:00:37 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
dbf438f99dcb184f7fc964005cf524fd48dba3d7
peft/examples/causal_language_modeling/peft_prefix_tuning_clm.ipynb
T

139 KiB
Raw Blame History

In [1]:
from transformers import AutoModelForCausalLM
from peft import get_peft_config,get_peft_model, get_peft_model_state_dict, set_peft_model_state_dict, PrefixTuningConfig, TaskType, peft_model_load_and_dispatch, bloom_model_postprocess_past_key_value, PeftType
import torch
from datasets import load_dataset
import os
from transformers import AutoTokenizer
from torch.utils.data import DataLoader
from transformers import default_data_collator,get_linear_schedule_with_warmup
from tqdm import tqdm
from datasets import load_dataset

device = "cuda"
model_name_or_path = "bigscience/bloomz-560m"
tokenizer_name_or_path = "bigscience/bloomz-560m"
peft_config = PrefixTuningConfig(task_type=TaskType.CAUSAL_LM, 
                                num_virtual_tokens=30, 
                                postprocess_past_key_value_function=bloom_model_postprocess_past_key_value)

dataset_name = "twitter_complaints"
checkpoint_name = f"{dataset_name}_{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}_v1.pt".replace("/", "_")
text_column = "Tweet text"
label_column = "text_label"
max_length=64
lr = 3e-2
num_epochs = 50
batch_size=8
In [2]:
from datasets import load_dataset

dataset = load_dataset("ought/raft", dataset_name)

classes = [k.replace("_", " ") for k in dataset["train"].features["Label"].names]
print(classes)
dataset = dataset.map(
    lambda x: {"text_label": [classes[label] for label in x["Label"]]},
    batched=True,
    num_proc=1,
    
)
print(dataset)
dataset["train"][0]
Out [2]:
Found cached dataset raft (/home/sourab/.cache/huggingface/datasets/ought___raft/twitter_complaints/1.1.0/79c4de1312c1e3730043f7db07179c914f48403101f7124e2fe336f6f54d9f84)

  0%|          | 0/2 [00:00<?, ?it/s]
Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/ought___raft/twitter_complaints/1.1.0/79c4de1312c1e3730043f7db07179c914f48403101f7124e2fe336f6f54d9f84/cache-05388978db6af01d.arrow
Loading cached processed dataset at /home/sourab/.cache/huggingface/datasets/ought___raft/twitter_complaints/1.1.0/79c4de1312c1e3730043f7db07179c914f48403101f7124e2fe336f6f54d9f84/cache-e3fade69c4ae889a.arrow

['Unlabeled', 'complaint', 'no complaint']
DatasetDict({
    train: Dataset({
        features: ['Tweet text', 'ID', 'Label', 'text_label'],
        num_rows: 50
    })
    test: Dataset({
        features: ['Tweet text', 'ID', 'Label', 'text_label'],
        num_rows: 3399
    })
})

{'Tweet text': '@HMRCcustomers No this is my first job',
 'ID': 0,
 'Label': 2,
 'text_label': 'no complaint'}
In [3]:
# data preprocessing
tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
if tokenizer.pad_token_id is None:
    tokenizer.pad_token_id = tokenizer.eos_token_id
target_max_length = max([len(tokenizer(class_label)["input_ids"]) for class_label in classes])
print(target_max_length)
def preprocess_function(examples):
    batch_size = len(examples[text_column])
    inputs = [f"{text_column} : {x} Label : " for x in examples[text_column]]
    targets = [str(x) for x in examples[label_column]]
    model_inputs = tokenizer(inputs)
    labels = tokenizer(targets)
    for i in range(batch_size):
        sample_input_ids = model_inputs["input_ids"][i]
        label_input_ids = labels["input_ids"][i] + [tokenizer.pad_token_id]
        #print(i, sample_input_ids, label_input_ids)
        model_inputs["input_ids"][i] = sample_input_ids + label_input_ids 
        labels["input_ids"][i] = [-100] * len(sample_input_ids) + label_input_ids
        model_inputs["attention_mask"][i] = [1] * len(model_inputs["input_ids"][i])
    #print(model_inputs)
    for i in range(batch_size):
        sample_input_ids = model_inputs["input_ids"][i]
        label_input_ids = labels["input_ids"][i]
        model_inputs["input_ids"][i] = [tokenizer.pad_token_id]*(max_length-len(sample_input_ids)) + sample_input_ids
        model_inputs["attention_mask"][i] = [0]*(max_length-len(sample_input_ids)) + model_inputs["attention_mask"][i]
        labels["input_ids"][i] =  [-100]*(max_length-len(sample_input_ids)) + label_input_ids 
        model_inputs["input_ids"][i] = torch.tensor(model_inputs["input_ids"][i][:max_length])
        model_inputs["attention_mask"][i] = torch.tensor(model_inputs["attention_mask"][i][:max_length])
        labels["input_ids"][i] = torch.tensor(labels["input_ids"][i][:max_length]) 
    model_inputs["labels"] = labels["input_ids"]
    return model_inputs



processed_datasets = dataset.map(
            preprocess_function,
            batched=True,
            num_proc=1,
            remove_columns=dataset["train"].column_names,
            load_from_cache_file=False,
            desc="Running tokenizer on dataset",
        )

train_dataset = processed_datasets["train"]
eval_dataset = processed_datasets["train"]


train_dataloader = DataLoader(
        train_dataset, shuffle=True, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True
    )
eval_dataloader = DataLoader(eval_dataset, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True)




    
3

Running tokenizer on dataset:   0%|          | 0/1 [00:00<?, ?ba/s]
Running tokenizer on dataset:   0%|          | 0/4 [00:00<?, ?ba/s]
In [4]:
def test_preprocess_function(examples):
    batch_size = len(examples[text_column])
    inputs = [f"{text_column} : {x} Label : " for x in examples[text_column]]
    model_inputs = tokenizer(inputs)
    #print(model_inputs)
    for i in range(batch_size):
        sample_input_ids = model_inputs["input_ids"][i]
        model_inputs["input_ids"][i] = [tokenizer.pad_token_id]*(max_length-len(sample_input_ids)) + sample_input_ids
        model_inputs["attention_mask"][i] = [0]*(max_length-len(sample_input_ids)) + model_inputs["attention_mask"][i]
        model_inputs["input_ids"][i] = torch.tensor(model_inputs["input_ids"][i][:max_length])
        model_inputs["attention_mask"][i] = torch.tensor(model_inputs["attention_mask"][i][:max_length])
    return model_inputs

test_dataset = dataset["test"].map(
            test_preprocess_function,
            batched=True,
            num_proc=1,
            remove_columns=dataset["train"].column_names,
            load_from_cache_file=False,
            desc="Running tokenizer on dataset",
        )

test_dataloader = DataLoader(test_dataset, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True)
next(iter(test_dataloader))
Out [4]:
Running tokenizer on dataset:   0%|          | 0/4 [00:00<?, ?ba/s]
{'input_ids': tensor([[     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
          227985,   5484,    915,   2566,  74757,  64626,  12384,  44639,    613,
           52282,   2670,  79920,   3344,   1002,    368,  17646,  14472,   8348,
             664,    718,      4,  19036,     17,  31849,     17,   6312,     76,
              44,  62470,     56,     91,     50,  14839,     21,  77658,    915,
             210],
         [     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3, 227985,   5484,    915,    405, 187059,
            2256,    664,   2550,  18833,  18607, 162467,      4,   1387,   6199,
            3291,  23405,    613,   4657,  17082,    566,   3432,    368,  78851,
            1185,  61273,  23181,   1553,  15596,    212, 116057,  77658,    915,
             210],
         [     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3, 227985,   5484,
             915,  39762,   2566,  22253,   6201,  75701,     15,    632,    718,
            5840,  10006,   6201,  18881,    427,   3804,  19528,    267, 158974,
            1320,    368,  10029,    632,  49666,     92,     34,  77658,    915,
             210],
         [     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3, 227985,   5484,    915,   2566, 104565,   8695,   2089,   6140,
          109676,  99579,   1369,    512,    368,   4570,     54,    632,    368,
            1503, 241485, 132226,     15,    982,    727,   1152,  18100,    861,
           32596,  77597, 168154,   1306, 132226,   4346,  87843,     17, 130462,
             364,  32923,     89,     53,   8309,     20,     75,  77658,    915,
             210],
         [     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3, 227985,   5484,    915,   2566,
           14173,   2960,  29906,    387,  20706,  49337,   1369,  77658,    915,
             210],
         [     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3, 227985,   5484,    915,   2566, 219553,  45736,
           36876,   1713,     72,    707, 187205,  13002, 177324,  77658,    915,
             210],
         [     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3, 227985,   5484,    915,   2566, 233938,  28518,  13716,
             427,  28146,   1119,  17918,     17, 236706,    368, 214997,   7555,
           48659,   5276,  21600,    343,     17,  51416,  22403,    318,   1531,
            1306,   1130,  20934,    567, 101161, 184849,  87843,     17,   1594,
           15231,   2052,  16642,     20,   7180,     80,     26,  77658,    915,
             210],
         [     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
          227985,   5484,    915,   2566,     80,   2068,    479,   2566,     80,
            1376,    878, 147587,   3904,    632,    368,   6084,  65673,  78851,
           11736,  15527,  19082,  33151,    461,     17,  45575,  17887,    632,
            5219,  14216,  68870,   5967,   1841,   4346,  87843,     17,   1594,
           14512,     27,     71,   8184,     19,    290,  63748,  77658,    915,
             210]]),
 'attention_mask': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
          0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
          0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
          0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}
In [5]:
next(iter(train_dataloader))
Out [5]:
{'input_ids': tensor([[     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3, 227985,   5484,    915,   2566,
           15157,   4867,  14731, 165189,   2021,    769,  11528,   7220,  35025,
             530,  27937, 149533,   1965,  43435, 163255,   1141,   3611,     17,
           30655,    632,   1119,     17,  77658,    915,    210,  16449,   5952,
               3],
         [     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3, 227985,   5484,    915,   5673,    473,
           11229,   2213,   2670,  35307,  28629,    461,   2566,   2765,   1531,
            3470,  47134,  10144,   2765,   1531,    427,   2909,  17918,   6782,
           27268,   4390,   1517,     17,   3904,    632,    267,   6497,    483,
             361,   2670, 101848,     17,  32465,   9585,   2566,     37,   2481,
            2566,     37,   2481,  12384,  77658,    915,    210,  16449,   5952,
               3],
         [     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3, 227985,   5484,    915,
            2566,     80,  11010,    905, 200058,   3904,   9746,   3370,    722,
            1074,     15,   1965,    600,  50713, 191765,   4973,     34, 200008,
          123467,   1306,   1427,  16198,   3262,  11700,  35237,  12602,   1293,
            8398,    530,   1999,   4346,  87843,     17,   1594,  35367, 241792,
          130376,    894,   3143,     42,  77658,    915,    210,   1936, 106863,
               3],
         [     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3, 227985,   5484,    915,   2566,     46,  30579,     46,
           12592,     57,    473,   3370, 199020,    267,  46019,  21302,  15804,
             361,  13300, 132548,  27761,  77658,    915,    210,   1936, 106863,
               3],
         [     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3, 227985,
            5484,    915, 191971,    261,  74182,      4, 226928,    427,  25608,
            1002,  39839,    473,   4472,   2550,     41,  86461,   4352,  19821,
            2550, 238683,  13663,  87843,     17,   1594,  14663,     36,     52,
              44,   6794,     81,  87781,  77658,    915,    210,   1936, 106863,
               3],
         [     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3, 227985,
            5484,    915,   2566,     49,    656,    266,  53312,   2566,     49,
             656,    266, 100800,  10966,    368,  62798,    632,    267, 190795,
          230331,  15226,    427,   2213,  20889,   5011,  20073,  13538,   5840,
          221781,  41051,  49337,  42696,  77658,    915,    210,  16449,   5952,
               3],
         [     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3, 227985,   5484,
             915,   2566,     44,    256,  67875,  21033,  86274,  79707,   2632,
            9999,    427,   2150,  54036,  98091,     34, 112164,  15971,  16154,
            5382,    861,   7220,     17,  77658,    915,    210,   1936, 106863,
               3],
         [     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3, 227985,   5484,    915,   2566, 181601,  62835,   6640,
            1935,    210,    654,  10144,    361,  21479,    300,  10583,   1369,
             632,    718,  86109,     17,  77658,    915,    210,  16449,   5952,
               3]]),
 'attention_mask': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
          0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
          0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
          0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]),
 'labels': tensor([[  -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,  16449,   5952,
               3],
         [  -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,  16449,   5952,
               3],
         [  -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   1936, 106863,
               3],
         [  -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   1936, 106863,
               3],
         [  -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   1936, 106863,
               3],
         [  -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,  16449,   5952,
               3],
         [  -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   1936, 106863,
               3],
         [  -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,   -100,
            -100,   -100,   -100,   -100,   -100,   -100,   -100,  16449,   5952,
               3]])}
In [6]:
len(test_dataloader)
Out [6]:
425
In [7]:
next(iter(test_dataloader))
Out [7]:
{'input_ids': tensor([[     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
          227985,   5484,    915,   2566,  74757,  64626,  12384,  44639,    613,
           52282,   2670,  79920,   3344,   1002,    368,  17646,  14472,   8348,
             664,    718,      4,  19036,     17,  31849,     17,   6312,     76,
              44,  62470,     56,     91,     50,  14839,     21,  77658,    915,
             210],
         [     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3, 227985,   5484,    915,    405, 187059,
            2256,    664,   2550,  18833,  18607, 162467,      4,   1387,   6199,
            3291,  23405,    613,   4657,  17082,    566,   3432,    368,  78851,
            1185,  61273,  23181,   1553,  15596,    212, 116057,  77658,    915,
             210],
         [     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3, 227985,   5484,
             915,  39762,   2566,  22253,   6201,  75701,     15,    632,    718,
            5840,  10006,   6201,  18881,    427,   3804,  19528,    267, 158974,
            1320,    368,  10029,    632,  49666,     92,     34,  77658,    915,
             210],
         [     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3, 227985,   5484,    915,   2566, 104565,   8695,   2089,   6140,
          109676,  99579,   1369,    512,    368,   4570,     54,    632,    368,
            1503, 241485, 132226,     15,    982,    727,   1152,  18100,    861,
           32596,  77597, 168154,   1306, 132226,   4346,  87843,     17, 130462,
             364,  32923,     89,     53,   8309,     20,     75,  77658,    915,
             210],
         [     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3, 227985,   5484,    915,   2566,
           14173,   2960,  29906,    387,  20706,  49337,   1369,  77658,    915,
             210],
         [     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3, 227985,   5484,    915,   2566, 219553,  45736,
           36876,   1713,     72,    707, 187205,  13002, 177324,  77658,    915,
             210],
         [     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3, 227985,   5484,    915,   2566, 233938,  28518,  13716,
             427,  28146,   1119,  17918,     17, 236706,    368, 214997,   7555,
           48659,   5276,  21600,    343,     17,  51416,  22403,    318,   1531,
            1306,   1130,  20934,    567, 101161, 184849,  87843,     17,   1594,
           15231,   2052,  16642,     20,   7180,     80,     26,  77658,    915,
             210],
         [     3,      3,      3,      3,      3,      3,      3,      3,      3,
               3,      3,      3,      3,      3,      3,      3,      3,      3,
          227985,   5484,    915,   2566,     80,   2068,    479,   2566,     80,
            1376,    878, 147587,   3904,    632,    368,   6084,  65673,  78851,
           11736,  15527,  19082,  33151,    461,     17,  45575,  17887,    632,
            5219,  14216,  68870,   5967,   1841,   4346,  87843,     17,   1594,
           14512,     27,     71,   8184,     19,    290,  63748,  77658,    915,
             210]]),
 'attention_mask': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
          0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
          0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
          0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
         [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}
In [8]:

# creating model
model = AutoModelForCausalLM.from_pretrained(model_name_or_path)
model = get_peft_model(model, peft_config)
model.print_trainable_parameters()


trainable params: 1474560 || all params: 560689152 || trainable%: 0.26299064191632515
In [9]:
model.print_trainable_parameters()
trainable params: 1474560 || all params: 560689152 || trainable%: 0.26299064191632515
In [10]:
model
Out [10]:
PETModelForCausalLM(
  (base_model): BloomForCausalLM(
    (transformer): BloomModel(
      (word_embeddings): Embedding(250880, 1024)
      (word_embeddings_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
      (h): ModuleList(
        (0): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (1): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (2): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (3): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (4): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (5): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (6): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (7): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (8): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (9): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (10): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (11): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (12): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (13): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (14): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (15): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (16): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (17): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (18): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (19): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (20): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (21): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (22): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (23): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
      )
      (ln_f): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
    )
    (lm_head): Linear(in_features=1024, out_features=250880, bias=False)
  )
  (word_embeddings): Embedding(250880, 1024)
  (prompt_encoder): PrefixEncoder(
    (embedding): Embedding(30, 49152)
  )
)
In [11]:
model.peft_config
Out [11]:
PrefixTuningConfig(pet_type=<PETType.PREFIX_TUNING: 'PREFIX_TUNING'>, task_type=<TaskType.CAUSAL_LM: 'CAUSAL_LM'>, inference_mode=False, num_virtual_tokens=30, token_dim=1024, num_transformer_submodules=1, num_attention_heads=16, num_layers=24, encoder_hidden_size=1024, prefix_projection=False, postprocess_past_key_value_function=<function bloom_model_postprocess_past_key_value at 0x7f6fddc456c0>)
In [12]:
# model
# optimizer and lr scheduler
optimizer = torch.optim.AdamW(model.parameters(), lr=lr)
lr_scheduler = get_linear_schedule_with_warmup(
    optimizer=optimizer,
    num_warmup_steps=0,
    num_training_steps=(len(train_dataloader) * num_epochs),
)
In [13]:
# training and evaluation
model = model.to(device)

for epoch in range(num_epochs):
    model.train()
    total_loss = 0
    for step, batch in enumerate(tqdm(train_dataloader)):
        batch = {k: v.to(device) for k, v in batch.items()}
#         print(batch)
#         print(batch["input_ids"].shape)
        outputs = model(**batch)
        loss = outputs.loss
        total_loss += loss.detach().float()
        loss.backward()
        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()

    model.eval()
    eval_loss = 0
    eval_preds = []
    for step, batch in enumerate(tqdm(eval_dataloader)):
        batch = {k: v.to(device) for k, v in batch.items()}
        with torch.no_grad():
            outputs = model(**batch)
        loss = outputs.loss
        eval_loss += loss.detach().float()
        eval_preds.extend(tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True))

    eval_epoch_loss = eval_loss/len(train_dataloader)
    eval_ppl = torch.exp(eval_epoch_loss)
    train_epoch_loss = total_loss/len(eval_dataloader)
    train_ppl = torch.exp(train_epoch_loss)
    print(f"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}")

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:01<00:00,  5.93it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.43it/s]

epoch=0: train_ppl=tensor(4.1394e+09, device='cuda:0') train_epoch_loss=tensor(22.1438, device='cuda:0') eval_ppl=tensor(1402.4835, device='cuda:0') eval_epoch_loss=tensor(7.2460, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.39it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.49it/s]

epoch=1: train_ppl=tensor(246.8958, device='cuda:0') train_epoch_loss=tensor(5.5090, device='cuda:0') eval_ppl=tensor(62.6347, device='cuda:0') eval_epoch_loss=tensor(4.1373, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.40it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]

epoch=2: train_ppl=tensor(40.7671, device='cuda:0') train_epoch_loss=tensor(3.7079, device='cuda:0') eval_ppl=tensor(20.4430, device='cuda:0') eval_epoch_loss=tensor(3.0176, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.37it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.52it/s]

epoch=3: train_ppl=tensor(13.3799, device='cuda:0') train_epoch_loss=tensor(2.5938, device='cuda:0') eval_ppl=tensor(7.8204, device='cuda:0') eval_epoch_loss=tensor(2.0567, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.44it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.55it/s]

epoch=4: train_ppl=tensor(5.6719, device='cuda:0') train_epoch_loss=tensor(1.7355, device='cuda:0') eval_ppl=tensor(3.2507, device='cuda:0') eval_epoch_loss=tensor(1.1789, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.44it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]

epoch=5: train_ppl=tensor(2.4837, device='cuda:0') train_epoch_loss=tensor(0.9098, device='cuda:0') eval_ppl=tensor(1.5463, device='cuda:0') eval_epoch_loss=tensor(0.4359, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.53it/s]

epoch=6: train_ppl=tensor(1.4864, device='cuda:0') train_epoch_loss=tensor(0.3964, device='cuda:0') eval_ppl=tensor(1.8123, device='cuda:0') eval_epoch_loss=tensor(0.5946, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.45it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.49it/s]

epoch=7: train_ppl=tensor(1.4421, device='cuda:0') train_epoch_loss=tensor(0.3661, device='cuda:0') eval_ppl=tensor(1.6831, device='cuda:0') eval_epoch_loss=tensor(0.5206, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.38it/s]

epoch=8: train_ppl=tensor(1.5938, device='cuda:0') train_epoch_loss=tensor(0.4661, device='cuda:0') eval_ppl=tensor(1.4380, device='cuda:0') eval_epoch_loss=tensor(0.3632, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.44it/s]

epoch=9: train_ppl=tensor(1.2076, device='cuda:0') train_epoch_loss=tensor(0.1886, device='cuda:0') eval_ppl=tensor(1.2875, device='cuda:0') eval_epoch_loss=tensor(0.2527, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.51it/s]

epoch=10: train_ppl=tensor(1.2741, device='cuda:0') train_epoch_loss=tensor(0.2422, device='cuda:0') eval_ppl=tensor(1.2635, device='cuda:0') eval_epoch_loss=tensor(0.2339, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]

epoch=11: train_ppl=tensor(1.2219, device='cuda:0') train_epoch_loss=tensor(0.2004, device='cuda:0') eval_ppl=tensor(1.1836, device='cuda:0') eval_epoch_loss=tensor(0.1686, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.52it/s]

epoch=12: train_ppl=tensor(1.1773, device='cuda:0') train_epoch_loss=tensor(0.1632, device='cuda:0') eval_ppl=tensor(1.1829, device='cuda:0') eval_epoch_loss=tensor(0.1680, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.41it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.53it/s]

epoch=13: train_ppl=tensor(1.1660, device='cuda:0') train_epoch_loss=tensor(0.1536, device='cuda:0') eval_ppl=tensor(1.1448, device='cuda:0') eval_epoch_loss=tensor(0.1353, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.51it/s]

epoch=14: train_ppl=tensor(1.1428, device='cuda:0') train_epoch_loss=tensor(0.1334, device='cuda:0') eval_ppl=tensor(1.1405, device='cuda:0') eval_epoch_loss=tensor(0.1315, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]

epoch=15: train_ppl=tensor(1.1204, device='cuda:0') train_epoch_loss=tensor(0.1137, device='cuda:0') eval_ppl=tensor(1.1171, device='cuda:0') eval_epoch_loss=tensor(0.1108, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.50it/s]

epoch=16: train_ppl=tensor(1.1038, device='cuda:0') train_epoch_loss=tensor(0.0988, device='cuda:0') eval_ppl=tensor(1.0856, device='cuda:0') eval_epoch_loss=tensor(0.0821, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.53it/s]

epoch=17: train_ppl=tensor(1.0810, device='cuda:0') train_epoch_loss=tensor(0.0779, device='cuda:0') eval_ppl=tensor(1.0642, device='cuda:0') eval_epoch_loss=tensor(0.0623, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.41it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]

epoch=18: train_ppl=tensor(1.0586, device='cuda:0') train_epoch_loss=tensor(0.0569, device='cuda:0') eval_ppl=tensor(1.0542, device='cuda:0') eval_epoch_loss=tensor(0.0528, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]

epoch=19: train_ppl=tensor(1.0449, device='cuda:0') train_epoch_loss=tensor(0.0439, device='cuda:0') eval_ppl=tensor(1.0455, device='cuda:0') eval_epoch_loss=tensor(0.0445, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]

epoch=20: train_ppl=tensor(1.0310, device='cuda:0') train_epoch_loss=tensor(0.0306, device='cuda:0') eval_ppl=tensor(1.0159, device='cuda:0') eval_epoch_loss=tensor(0.0158, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.42it/s]

epoch=21: train_ppl=tensor(1.0200, device='cuda:0') train_epoch_loss=tensor(0.0198, device='cuda:0') eval_ppl=tensor(1.0333, device='cuda:0') eval_epoch_loss=tensor(0.0327, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.46it/s]

epoch=22: train_ppl=tensor(1.0209, device='cuda:0') train_epoch_loss=tensor(0.0207, device='cuda:0') eval_ppl=tensor(1.0187, device='cuda:0') eval_epoch_loss=tensor(0.0185, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.40it/s]

epoch=23: train_ppl=tensor(1.0098, device='cuda:0') train_epoch_loss=tensor(0.0097, device='cuda:0') eval_ppl=tensor(1.0097, device='cuda:0') eval_epoch_loss=tensor(0.0096, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.45it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.50it/s]

epoch=24: train_ppl=tensor(1.0120, device='cuda:0') train_epoch_loss=tensor(0.0120, device='cuda:0') eval_ppl=tensor(1.0162, device='cuda:0') eval_epoch_loss=tensor(0.0161, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.49it/s]

epoch=25: train_ppl=tensor(1.0120, device='cuda:0') train_epoch_loss=tensor(0.0119, device='cuda:0') eval_ppl=tensor(1.0090, device='cuda:0') eval_epoch_loss=tensor(0.0090, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.40it/s]

epoch=26: train_ppl=tensor(1.0125, device='cuda:0') train_epoch_loss=tensor(0.0124, device='cuda:0') eval_ppl=tensor(1.0079, device='cuda:0') eval_epoch_loss=tensor(0.0079, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]

epoch=27: train_ppl=tensor(1.0072, device='cuda:0') train_epoch_loss=tensor(0.0072, device='cuda:0') eval_ppl=tensor(1.0043, device='cuda:0') eval_epoch_loss=tensor(0.0043, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.44it/s]

epoch=28: train_ppl=tensor(1.0044, device='cuda:0') train_epoch_loss=tensor(0.0044, device='cuda:0') eval_ppl=tensor(1.0049, device='cuda:0') eval_epoch_loss=tensor(0.0048, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.51it/s]

epoch=29: train_ppl=tensor(1.0047, device='cuda:0') train_epoch_loss=tensor(0.0047, device='cuda:0') eval_ppl=tensor(1.0038, device='cuda:0') eval_epoch_loss=tensor(0.0038, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.41it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.52it/s]

epoch=30: train_ppl=tensor(1.0037, device='cuda:0') train_epoch_loss=tensor(0.0037, device='cuda:0') eval_ppl=tensor(1.0035, device='cuda:0') eval_epoch_loss=tensor(0.0035, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]

epoch=31: train_ppl=tensor(1.0033, device='cuda:0') train_epoch_loss=tensor(0.0033, device='cuda:0') eval_ppl=tensor(1.0031, device='cuda:0') eval_epoch_loss=tensor(0.0031, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]

epoch=32: train_ppl=tensor(1.0029, device='cuda:0') train_epoch_loss=tensor(0.0029, device='cuda:0') eval_ppl=tensor(1.0029, device='cuda:0') eval_epoch_loss=tensor(0.0029, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.45it/s]

epoch=33: train_ppl=tensor(1.0027, device='cuda:0') train_epoch_loss=tensor(0.0027, device='cuda:0') eval_ppl=tensor(1.0028, device='cuda:0') eval_epoch_loss=tensor(0.0028, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.50it/s]

epoch=34: train_ppl=tensor(1.0026, device='cuda:0') train_epoch_loss=tensor(0.0026, device='cuda:0') eval_ppl=tensor(1.0027, device='cuda:0') eval_epoch_loss=tensor(0.0027, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.49it/s]

epoch=35: train_ppl=tensor(1.0026, device='cuda:0') train_epoch_loss=tensor(0.0026, device='cuda:0') eval_ppl=tensor(1.0026, device='cuda:0') eval_epoch_loss=tensor(0.0026, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.30it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.03it/s]

epoch=36: train_ppl=tensor(1.0025, device='cuda:0') train_epoch_loss=tensor(0.0025, device='cuda:0') eval_ppl=tensor(1.0025, device='cuda:0') eval_epoch_loss=tensor(0.0025, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]

epoch=37: train_ppl=tensor(1.0024, device='cuda:0') train_epoch_loss=tensor(0.0023, device='cuda:0') eval_ppl=tensor(1.0024, device='cuda:0') eval_epoch_loss=tensor(0.0024, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.40it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]

epoch=38: train_ppl=tensor(1.0023, device='cuda:0') train_epoch_loss=tensor(0.0023, device='cuda:0') eval_ppl=tensor(1.0023, device='cuda:0') eval_epoch_loss=tensor(0.0023, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]

epoch=39: train_ppl=tensor(1.0023, device='cuda:0') train_epoch_loss=tensor(0.0023, device='cuda:0') eval_ppl=tensor(1.0023, device='cuda:0') eval_epoch_loss=tensor(0.0023, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.51it/s]

epoch=40: train_ppl=tensor(1.0022, device='cuda:0') train_epoch_loss=tensor(0.0022, device='cuda:0') eval_ppl=tensor(1.0023, device='cuda:0') eval_epoch_loss=tensor(0.0022, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.46it/s]

epoch=41: train_ppl=tensor(1.0022, device='cuda:0') train_epoch_loss=tensor(0.0022, device='cuda:0') eval_ppl=tensor(1.0022, device='cuda:0') eval_epoch_loss=tensor(0.0022, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.43it/s]

epoch=42: train_ppl=tensor(1.0021, device='cuda:0') train_epoch_loss=tensor(0.0021, device='cuda:0') eval_ppl=tensor(1.0022, device='cuda:0') eval_epoch_loss=tensor(0.0022, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.49it/s]

epoch=43: train_ppl=tensor(1.0022, device='cuda:0') train_epoch_loss=tensor(0.0022, device='cuda:0') eval_ppl=tensor(1.0021, device='cuda:0') eval_epoch_loss=tensor(0.0021, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.46it/s]

epoch=44: train_ppl=tensor(1.0021, device='cuda:0') train_epoch_loss=tensor(0.0021, device='cuda:0') eval_ppl=tensor(1.0021, device='cuda:0') eval_epoch_loss=tensor(0.0021, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.40it/s]

epoch=45: train_ppl=tensor(1.0020, device='cuda:0') train_epoch_loss=tensor(0.0020, device='cuda:0') eval_ppl=tensor(1.0021, device='cuda:0') eval_epoch_loss=tensor(0.0021, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.41it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.43it/s]

epoch=46: train_ppl=tensor(1.0021, device='cuda:0') train_epoch_loss=tensor(0.0021, device='cuda:0') eval_ppl=tensor(1.0021, device='cuda:0') eval_epoch_loss=tensor(0.0021, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.40it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.45it/s]

epoch=47: train_ppl=tensor(1.0021, device='cuda:0') train_epoch_loss=tensor(0.0021, device='cuda:0') eval_ppl=tensor(1.0021, device='cuda:0') eval_epoch_loss=tensor(0.0021, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.41it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.43it/s]

epoch=48: train_ppl=tensor(1.0020, device='cuda:0') train_epoch_loss=tensor(0.0020, device='cuda:0') eval_ppl=tensor(1.0021, device='cuda:0') eval_epoch_loss=tensor(0.0021, device='cuda:0')

100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.41it/s]
100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.42it/s]
epoch=49: train_ppl=tensor(1.0020, device='cuda:0') train_epoch_loss=tensor(0.0020, device='cuda:0') eval_ppl=tensor(1.0021, device='cuda:0') eval_epoch_loss=tensor(0.0021, device='cuda:0')

In [14]:
model.eval()
i = 12
inputs = tokenizer(f'{text_column} : {dataset["test"][i]["Tweet text"]} Label : ', return_tensors="pt")
print(dataset["test"][i]["Tweet text"])
print(inputs)

with torch.no_grad():
    inputs = {k: v.to(device) for k, v in inputs.items()}
    outputs = model.generate(input_ids=inputs["input_ids"], attention_mask=inputs["attention_mask"], max_new_tokens=10)
    print(outputs)
    print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))
    
@VW @QuirkCars some unauthorized work done on my engine now throwing a check engine light about a week later. Very upset.
{'input_ids': tensor([[227985,   5484,    915,   2566,     57,     58,   2566,   5232, 132511,
             38,   4599,   3331,   1035, 192352,   2909,  11541,    664,   2670,
          22218,   5840, 108218,    267,   7010,  22218,  12490,   3638,    267,
          14319,  10494,     17,  93269, 123055,     17,  77658,    915,    210]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}
tensor([[227985,   5484,    915,   2566,     57,     58,   2566,   5232, 132511,
             38,   4599,   3331,   1035, 192352,   2909,  11541,    664,   2670,
          22218,   5840, 108218,    267,   7010,  22218,  12490,   3638,    267,
          14319,  10494,     17,  93269, 123055,     17,  77658,    915,    210,
          16449,   5952,      3,      3,      3,      3,      3,      3,      3,
              3]], device='cuda:0')
['Tweet text : @VW @QuirkCars some unauthorized work done on my engine now throwing a check engine light about a week later. Very upset. Label : complaint']

In [ ]:
model.eval()
eval_loss = 0
eval_preds = []
for step, batch in enumerate(tqdm(test_dataloader)):
    batch = {k: v.to(device) for k, v in batch.items() if k!="labels"}
    with torch.no_grad():
        outputs = model.generate(**batch, max_new_tokens=10)
    eval_preds.extend(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))


In [15]:
# saving model
state_dict = get_peft_model_state_dict(model)
torch.save(state_dict, checkpoint_name)
print(state_dict)
{'prompt_embeddings': tensor([[ 0.7356, -1.0849, -0.4560,  ...,  1.0242,  0.3908, -0.8000],
        [-1.5587, -0.3595,  0.1289,  ...,  0.5427,  1.0976,  1.7641],
        [-0.2113, -1.4675, -0.5976,  ...,  0.1691, -0.5843, -0.2658],
        ...,
        [ 2.1275,  0.7253,  0.0323,  ..., -1.2285, -0.5614,  0.0370],
        [-0.2258, -1.5149,  0.0685,  ..., -1.4476, -0.1348, -0.6910],
        [ 0.9089,  0.3947, -1.5271,  ...,  1.9079,  0.6473,  0.7306]])}

In [16]:
!du -h $checkpoint_name
huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
To disable this warning, you can either:
	- Avoid using `tokenizers` before the fork if possible
	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
5,7M	twitter_complaints_bigscience_bloomz-560m_PREFIX_TUNING_CAUSAL_LM_v1.pt

In [8]:
max_memory={0: "1GIB", 1: "1GIB", 2: "2GIB", 3: "2GIB", "cpu":"30GB"}

peft_config.inference_mode = True
print(peft_config)
model = AutoModelForCausalLM.from_pretrained(model_name_or_path, device_map="auto", max_memory=max_memory)
model = peft_model_load_and_dispatch(model, torch.load(checkpoint_name), peft_config, max_memory)


PrefixTuningConfig(pet_type=<PETType.PREFIX_TUNING: 'PREFIX_TUNING'>, task_type=<TaskType.CAUSAL_LM: 'CAUSAL_LM'>, inference_mode=True, num_virtual_tokens=30, token_dim=None, num_transformer_submodules=1, num_attention_heads=None, num_layers=None, encoder_hidden_size=None, prefix_projection=False, postprocess_past_key_value_function=<function bloom_model_postprocess_past_key_value at 0x7f913d791630>)
trainable params: 1474560 || all params: 560689152 || trainable%: 0.26299064191632515

In [9]:
model
Out [9]:
PETModelForCausalLM(
  (base_model): BloomForCausalLM(
    (transformer): BloomModel(
      (word_embeddings): Embedding(250880, 1024)
      (word_embeddings_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
      (h): ModuleList(
        (0): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (1): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (2): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (3): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (4): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (5): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (6): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (7): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (8): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (9): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (10): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (11): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (12): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (13): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (14): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (15): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (16): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (17): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (18): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (19): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (20): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (21): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (22): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
        (23): BloomBlock(
          (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (self_attention): BloomAttention(
            (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)
            (dense): Linear(in_features=1024, out_features=1024, bias=True)
            (attention_dropout): Dropout(p=0.0, inplace=False)
          )
          (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
          (mlp): BloomMLP(
            (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)
            (gelu_impl): BloomGelu()
            (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)
          )
        )
      )
      (ln_f): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)
    )
    (lm_head): Linear(in_features=1024, out_features=250880, bias=False)
  )
  (word_embeddings): Embedding(250880, 1024)
  (prompt_encoder): PrefixEncoder(
    (embedding): Embedding(30, 49152)
  )
)
In [10]:
model.hf_device_map
Out [10]:
{'base_model.transformer.word_embeddings': 3,
 'word_embeddings': 3,
 'base_model.transformer.word_embeddings_layernorm': 3,
 'base_model.transformer.h.0': 3,
 'base_model.transformer.h.1': 3,
 'base_model.transformer.h.2': 3,
 'base_model.transformer.h.3': 3,
 'base_model.transformer.h.4': 3,
 'base_model.transformer.h.5': 3,
 'base_model.transformer.h.6': 3,
 'base_model.transformer.h.7': 3,
 'base_model.transformer.h.8': 3,
 'base_model.transformer.h.9': 3,
 'base_model.transformer.h.10': 3,
 'base_model.transformer.h.11': 3,
 'base_model.transformer.h.12': 3,
 'base_model.transformer.h.13': 3,
 'base_model.transformer.h.14': 3,
 'base_model.transformer.h.15': 3,
 'base_model.transformer.h.16': 3,
 'base_model.transformer.h.17': 3,
 'base_model.transformer.h.18': 3,
 'base_model.transformer.h.19': 3,
 'base_model.transformer.h.20': 3,
 'base_model.transformer.h.21': 3,
 'base_model.transformer.h.22': 'cpu',
 'base_model.transformer.h.23': 'cpu',
 'base_model.transformer.ln_f': 'cpu',
 'base_model.lm_head': 'cpu',
 'prompt_encoder': 'cpu'}
In [11]:
model.eval()
i = 12
inputs = tokenizer(f'{text_column} : {dataset["test"][i]["Tweet text"]} Label : ', return_tensors="pt")
print(dataset["test"][i]["Tweet text"])
print(inputs)

with torch.no_grad():
    inputs = {k: v.to(device) for k, v in inputs.items()}
    outputs = model.generate(input_ids=inputs["input_ids"], attention_mask=inputs["attention_mask"], max_new_tokens=10)
    print(outputs)
    print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))

@VW @QuirkCars some unauthorized work done on my engine now throwing a check engine light about a week later. Very upset.
{'input_ids': tensor([[227985,   5484,    915,   2566,     57,     58,   2566,   5232, 132511,
             38,   4599,   3331,   1035, 192352,   2909,  11541,    664,   2670,
          22218,   5840, 108218,    267,   7010,  22218,  12490,   3638,    267,
          14319,  10494,     17,  93269, 123055,     17,  77658,    915,    210]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}
tensor([[227985,   5484,    915,   2566,     57,     58,   2566,   5232, 132511,
             38,   4599,   3331,   1035, 192352,   2909,  11541,    664,   2670,
          22218,   5840, 108218,    267,   7010,  22218,  12490,   3638,    267,
          14319,  10494,     17,  93269, 123055,     17,  77658,    915,    210,
          16449,   5952,      3,      3,      3,      3,      3,      3,      3,
              3]], device='cuda:0')
['Tweet text : @VW @QuirkCars some unauthorized work done on my engine now throwing a check engine light about a week later. Very upset. Label : complaint']

In [ ]:

			
		


		

			
			Reference in New Issue
			
			View Git Blame
			Copy Permalink