mirror of
https://github.com/wassname/peft.git
synced 2026-09-09 11:28:32 +08:00
Merge pull request #95 from huggingface/smangrul/add-whisper-example
adding whisper large peft+int8 training example
This commit is contained in:
Sourab Mangrulkar
+46
-40
@@ -30,7 +30,7 @@
|
||||
"import os\n",
|
||||
"from transformers import AutoTokenizer\n",
|
||||
"from torch.utils.data import DataLoader\n",
|
||||
"from transformers import default_data_collator,get_linear_schedule_with_warmup\n",
|
||||
"from transformers import default_data_collator, get_linear_schedule_with_warmup\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"\n",
|
||||
@@ -40,10 +40,10 @@
|
||||
"dataset_name = \"twitter_complaints\"\n",
|
||||
"text_column = \"Tweet text\"\n",
|
||||
"label_column = \"text_label\"\n",
|
||||
"max_length=64\n",
|
||||
"max_length = 64\n",
|
||||
"lr = 1e-3\n",
|
||||
"num_epochs = 50\n",
|
||||
"batch_size=8\n"
|
||||
"batch_size = 8"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -63,7 +63,6 @@
|
||||
" lambda x: {\"text_label\": [classes[label] for label in x[\"Label\"]]},\n",
|
||||
" batched=True,\n",
|
||||
" num_proc=1,\n",
|
||||
" \n",
|
||||
")\n",
|
||||
"print(dataset)\n",
|
||||
"dataset[\"train\"][0]"
|
||||
@@ -118,6 +117,8 @@
|
||||
" tokenizer.pad_token_id = tokenizer.eos_token_id\n",
|
||||
"target_max_length = max([len(tokenizer(class_label)[\"input_ids\"]) for class_label in classes])\n",
|
||||
"print(target_max_length)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def preprocess_function(examples):\n",
|
||||
" batch_size = len(examples[text_column])\n",
|
||||
" inputs = [f\"{text_column} : {x} Label : \" for x in examples[text_column]]\n",
|
||||
@@ -127,44 +128,43 @@
|
||||
" for i in range(batch_size):\n",
|
||||
" sample_input_ids = model_inputs[\"input_ids\"][i]\n",
|
||||
" label_input_ids = labels[\"input_ids\"][i] + [tokenizer.pad_token_id]\n",
|
||||
" #print(i, sample_input_ids, label_input_ids)\n",
|
||||
" model_inputs[\"input_ids\"][i] = sample_input_ids + label_input_ids \n",
|
||||
" # print(i, sample_input_ids, label_input_ids)\n",
|
||||
" model_inputs[\"input_ids\"][i] = sample_input_ids + label_input_ids\n",
|
||||
" labels[\"input_ids\"][i] = [-100] * len(sample_input_ids) + label_input_ids\n",
|
||||
" model_inputs[\"attention_mask\"][i] = [1] * len(model_inputs[\"input_ids\"][i])\n",
|
||||
" #print(model_inputs)\n",
|
||||
" # print(model_inputs)\n",
|
||||
" for i in range(batch_size):\n",
|
||||
" sample_input_ids = model_inputs[\"input_ids\"][i]\n",
|
||||
" label_input_ids = labels[\"input_ids\"][i]\n",
|
||||
" model_inputs[\"input_ids\"][i] = [tokenizer.pad_token_id]*(max_length-len(sample_input_ids)) + sample_input_ids\n",
|
||||
" model_inputs[\"attention_mask\"][i] = [0]*(max_length-len(sample_input_ids)) + model_inputs[\"attention_mask\"][i]\n",
|
||||
" labels[\"input_ids\"][i] = [-100]*(max_length-len(sample_input_ids)) + label_input_ids \n",
|
||||
" model_inputs[\"input_ids\"][i] = [tokenizer.pad_token_id] * (\n",
|
||||
" max_length - len(sample_input_ids)\n",
|
||||
" ) + sample_input_ids\n",
|
||||
" model_inputs[\"attention_mask\"][i] = [0] * (max_length - len(sample_input_ids)) + model_inputs[\n",
|
||||
" \"attention_mask\"\n",
|
||||
" ][i]\n",
|
||||
" labels[\"input_ids\"][i] = [-100] * (max_length - len(sample_input_ids)) + label_input_ids\n",
|
||||
" model_inputs[\"input_ids\"][i] = torch.tensor(model_inputs[\"input_ids\"][i][:max_length])\n",
|
||||
" model_inputs[\"attention_mask\"][i] = torch.tensor(model_inputs[\"attention_mask\"][i][:max_length])\n",
|
||||
" labels[\"input_ids\"][i] = torch.tensor(labels[\"input_ids\"][i][:max_length]) \n",
|
||||
" labels[\"input_ids\"][i] = torch.tensor(labels[\"input_ids\"][i][:max_length])\n",
|
||||
" model_inputs[\"labels\"] = labels[\"input_ids\"]\n",
|
||||
" return model_inputs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"processed_datasets = dataset.map(\n",
|
||||
" preprocess_function,\n",
|
||||
" batched=True,\n",
|
||||
" num_proc=1,\n",
|
||||
" remove_columns=dataset[\"train\"].column_names,\n",
|
||||
" load_from_cache_file=False,\n",
|
||||
" desc=\"Running tokenizer on dataset\",\n",
|
||||
" )\n",
|
||||
" preprocess_function,\n",
|
||||
" batched=True,\n",
|
||||
" num_proc=1,\n",
|
||||
" remove_columns=dataset[\"train\"].column_names,\n",
|
||||
" load_from_cache_file=False,\n",
|
||||
" desc=\"Running tokenizer on dataset\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"train_dataset = processed_datasets[\"train\"]\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"train_dataloader = DataLoader(\n",
|
||||
" train_dataset, shuffle=True, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" "
|
||||
" train_dataset, shuffle=True, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -178,23 +178,28 @@
|
||||
" batch_size = len(examples[text_column])\n",
|
||||
" inputs = [f\"{text_column} : {x} Label : \" for x in examples[text_column]]\n",
|
||||
" model_inputs = tokenizer(inputs)\n",
|
||||
" #print(model_inputs)\n",
|
||||
" # print(model_inputs)\n",
|
||||
" for i in range(batch_size):\n",
|
||||
" sample_input_ids = model_inputs[\"input_ids\"][i]\n",
|
||||
" model_inputs[\"input_ids\"][i] = [tokenizer.pad_token_id]*(max_length-len(sample_input_ids)) + sample_input_ids\n",
|
||||
" model_inputs[\"attention_mask\"][i] = [0]*(max_length-len(sample_input_ids)) + model_inputs[\"attention_mask\"][i]\n",
|
||||
" model_inputs[\"input_ids\"][i] = [tokenizer.pad_token_id] * (\n",
|
||||
" max_length - len(sample_input_ids)\n",
|
||||
" ) + sample_input_ids\n",
|
||||
" model_inputs[\"attention_mask\"][i] = [0] * (max_length - len(sample_input_ids)) + model_inputs[\n",
|
||||
" \"attention_mask\"\n",
|
||||
" ][i]\n",
|
||||
" model_inputs[\"input_ids\"][i] = torch.tensor(model_inputs[\"input_ids\"][i][:max_length])\n",
|
||||
" model_inputs[\"attention_mask\"][i] = torch.tensor(model_inputs[\"attention_mask\"][i][:max_length])\n",
|
||||
" return model_inputs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"processed_datasets = dataset.map(\n",
|
||||
" test_preprocess_function,\n",
|
||||
" batched=True,\n",
|
||||
" num_proc=1,\n",
|
||||
" remove_columns=dataset[\"train\"].column_names,\n",
|
||||
" load_from_cache_file=False,\n",
|
||||
" desc=\"Running tokenizer on dataset\",\n",
|
||||
" )\n",
|
||||
" test_preprocess_function,\n",
|
||||
" batched=True,\n",
|
||||
" num_proc=1,\n",
|
||||
" remove_columns=dataset[\"train\"].column_names,\n",
|
||||
" load_from_cache_file=False,\n",
|
||||
" desc=\"Running tokenizer on dataset\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"eval_dataset = processed_datasets[\"train\"]\n",
|
||||
"test_dataset = processed_datasets[\"test\"]\n",
|
||||
@@ -236,7 +241,8 @@
|
||||
],
|
||||
"source": [
|
||||
"from peft import PeftModel, PeftConfig\n",
|
||||
"max_memory={0: \"1GIB\", 1: \"1GIB\", 2: \"2GIB\", 3: \"10GIB\", \"cpu\":\"30GB\"}\n",
|
||||
"\n",
|
||||
"max_memory = {0: \"1GIB\", 1: \"1GIB\", 2: \"2GIB\", 3: \"10GIB\", \"cpu\": \"30GB\"}\n",
|
||||
"peft_model_id = \"smangrul/twitter_complaints_bigscience_bloomz-7b1_LORA_CAUSAL_LM\"\n",
|
||||
"\n",
|
||||
"config = PeftConfig.from_pretrained(peft_model_id)\n",
|
||||
@@ -251,7 +257,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#model"
|
||||
"# model"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -343,7 +349,7 @@
|
||||
"with torch.no_grad():\n",
|
||||
" outputs = model.generate(input_ids=inputs[\"input_ids\"], max_new_tokens=10)\n",
|
||||
" print(outputs)\n",
|
||||
" print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))\n"
|
||||
" print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -397,7 +403,7 @@
|
||||
"accuracy = correct / total * 100\n",
|
||||
"print(f\"{accuracy=}\")\n",
|
||||
"print(f\"{eval_preds[:10]=}\")\n",
|
||||
"print(f\"{dataset['train'][label_column][:10]=}\")\n"
|
||||
"print(f\"{dataset['train'][label_column][:10]=}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -416,7 +422,7 @@
|
||||
" outputs = model.generate(**batch, max_new_tokens=10)\n",
|
||||
" preds = outputs[:, max_length:].detach().cpu().numpy()\n",
|
||||
" test_preds.extend(tokenizer.batch_decode(preds, skip_special_tokens=True))\n",
|
||||
" if len(test_preds)>100:\n",
|
||||
" if len(test_preds) > 100:\n",
|
||||
" break\n",
|
||||
"test_preds"
|
||||
]
|
||||
|
||||
@@ -8,31 +8,31 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from transformers import AutoModelForCausalLM\n",
|
||||
"from peft import get_peft_config,get_peft_model, PrefixTuningConfig, TaskType, PeftType\n",
|
||||
"from peft import get_peft_config, get_peft_model, PrefixTuningConfig, TaskType, PeftType\n",
|
||||
"import torch\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"import os\n",
|
||||
"from transformers import AutoTokenizer\n",
|
||||
"from torch.utils.data import DataLoader\n",
|
||||
"from transformers import default_data_collator,get_linear_schedule_with_warmup\n",
|
||||
"from transformers import default_data_collator, get_linear_schedule_with_warmup\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"\n",
|
||||
"device = \"cuda\"\n",
|
||||
"model_name_or_path = \"bigscience/bloomz-560m\"\n",
|
||||
"tokenizer_name_or_path = \"bigscience/bloomz-560m\"\n",
|
||||
"peft_config = PrefixTuningConfig(task_type=TaskType.CAUSAL_LM, \n",
|
||||
" num_virtual_tokens=30)\n",
|
||||
"peft_config = PrefixTuningConfig(task_type=TaskType.CAUSAL_LM, num_virtual_tokens=30)\n",
|
||||
"\n",
|
||||
"dataset_name = \"twitter_complaints\"\n",
|
||||
"checkpoint_name = f\"{dataset_name}_{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}_v1.pt\".replace(\"/\", \"_\")\n",
|
||||
"checkpoint_name = f\"{dataset_name}_{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}_v1.pt\".replace(\n",
|
||||
" \"/\", \"_\"\n",
|
||||
")\n",
|
||||
"text_column = \"Tweet text\"\n",
|
||||
"label_column = \"text_label\"\n",
|
||||
"max_length=64\n",
|
||||
"max_length = 64\n",
|
||||
"lr = 3e-2\n",
|
||||
"num_epochs = 50\n",
|
||||
"batch_size=8\n",
|
||||
"\n"
|
||||
"batch_size = 8"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -112,7 +112,6 @@
|
||||
" lambda x: {\"text_label\": [classes[label] for label in x[\"Label\"]]},\n",
|
||||
" batched=True,\n",
|
||||
" num_proc=1,\n",
|
||||
" \n",
|
||||
")\n",
|
||||
"print(dataset)\n",
|
||||
"dataset[\"train\"][0]"
|
||||
@@ -167,6 +166,8 @@
|
||||
" tokenizer.pad_token_id = tokenizer.eos_token_id\n",
|
||||
"target_max_length = max([len(tokenizer(class_label)[\"input_ids\"]) for class_label in classes])\n",
|
||||
"print(target_max_length)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def preprocess_function(examples):\n",
|
||||
" batch_size = len(examples[text_column])\n",
|
||||
" inputs = [f\"{text_column} : {x} Label : \" for x in examples[text_column]]\n",
|
||||
@@ -176,47 +177,45 @@
|
||||
" for i in range(batch_size):\n",
|
||||
" sample_input_ids = model_inputs[\"input_ids\"][i]\n",
|
||||
" label_input_ids = labels[\"input_ids\"][i] + [tokenizer.pad_token_id]\n",
|
||||
" #print(i, sample_input_ids, label_input_ids)\n",
|
||||
" model_inputs[\"input_ids\"][i] = sample_input_ids + label_input_ids \n",
|
||||
" # print(i, sample_input_ids, label_input_ids)\n",
|
||||
" model_inputs[\"input_ids\"][i] = sample_input_ids + label_input_ids\n",
|
||||
" labels[\"input_ids\"][i] = [-100] * len(sample_input_ids) + label_input_ids\n",
|
||||
" model_inputs[\"attention_mask\"][i] = [1] * len(model_inputs[\"input_ids\"][i])\n",
|
||||
" #print(model_inputs)\n",
|
||||
" # print(model_inputs)\n",
|
||||
" for i in range(batch_size):\n",
|
||||
" sample_input_ids = model_inputs[\"input_ids\"][i]\n",
|
||||
" label_input_ids = labels[\"input_ids\"][i]\n",
|
||||
" model_inputs[\"input_ids\"][i] = [tokenizer.pad_token_id]*(max_length-len(sample_input_ids)) + sample_input_ids\n",
|
||||
" model_inputs[\"attention_mask\"][i] = [0]*(max_length-len(sample_input_ids)) + model_inputs[\"attention_mask\"][i]\n",
|
||||
" labels[\"input_ids\"][i] = [-100]*(max_length-len(sample_input_ids)) + label_input_ids \n",
|
||||
" model_inputs[\"input_ids\"][i] = [tokenizer.pad_token_id] * (\n",
|
||||
" max_length - len(sample_input_ids)\n",
|
||||
" ) + sample_input_ids\n",
|
||||
" model_inputs[\"attention_mask\"][i] = [0] * (max_length - len(sample_input_ids)) + model_inputs[\n",
|
||||
" \"attention_mask\"\n",
|
||||
" ][i]\n",
|
||||
" labels[\"input_ids\"][i] = [-100] * (max_length - len(sample_input_ids)) + label_input_ids\n",
|
||||
" model_inputs[\"input_ids\"][i] = torch.tensor(model_inputs[\"input_ids\"][i][:max_length])\n",
|
||||
" model_inputs[\"attention_mask\"][i] = torch.tensor(model_inputs[\"attention_mask\"][i][:max_length])\n",
|
||||
" labels[\"input_ids\"][i] = torch.tensor(labels[\"input_ids\"][i][:max_length]) \n",
|
||||
" labels[\"input_ids\"][i] = torch.tensor(labels[\"input_ids\"][i][:max_length])\n",
|
||||
" model_inputs[\"labels\"] = labels[\"input_ids\"]\n",
|
||||
" return model_inputs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"processed_datasets = dataset.map(\n",
|
||||
" preprocess_function,\n",
|
||||
" batched=True,\n",
|
||||
" num_proc=1,\n",
|
||||
" remove_columns=dataset[\"train\"].column_names,\n",
|
||||
" load_from_cache_file=False,\n",
|
||||
" desc=\"Running tokenizer on dataset\",\n",
|
||||
" )\n",
|
||||
" preprocess_function,\n",
|
||||
" batched=True,\n",
|
||||
" num_proc=1,\n",
|
||||
" remove_columns=dataset[\"train\"].column_names,\n",
|
||||
" load_from_cache_file=False,\n",
|
||||
" desc=\"Running tokenizer on dataset\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"train_dataset = processed_datasets[\"train\"]\n",
|
||||
"eval_dataset = processed_datasets[\"train\"]\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"train_dataloader = DataLoader(\n",
|
||||
" train_dataset, shuffle=True, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True\n",
|
||||
" )\n",
|
||||
"eval_dataloader = DataLoader(eval_dataset, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" "
|
||||
" train_dataset, shuffle=True, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True\n",
|
||||
")\n",
|
||||
"eval_dataloader = DataLoader(eval_dataset, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -230,23 +229,28 @@
|
||||
" batch_size = len(examples[text_column])\n",
|
||||
" inputs = [f\"{text_column} : {x} Label : \" for x in examples[text_column]]\n",
|
||||
" model_inputs = tokenizer(inputs)\n",
|
||||
" #print(model_inputs)\n",
|
||||
" # print(model_inputs)\n",
|
||||
" for i in range(batch_size):\n",
|
||||
" sample_input_ids = model_inputs[\"input_ids\"][i]\n",
|
||||
" model_inputs[\"input_ids\"][i] = [tokenizer.pad_token_id]*(max_length-len(sample_input_ids)) + sample_input_ids\n",
|
||||
" model_inputs[\"attention_mask\"][i] = [0]*(max_length-len(sample_input_ids)) + model_inputs[\"attention_mask\"][i]\n",
|
||||
" model_inputs[\"input_ids\"][i] = [tokenizer.pad_token_id] * (\n",
|
||||
" max_length - len(sample_input_ids)\n",
|
||||
" ) + sample_input_ids\n",
|
||||
" model_inputs[\"attention_mask\"][i] = [0] * (max_length - len(sample_input_ids)) + model_inputs[\n",
|
||||
" \"attention_mask\"\n",
|
||||
" ][i]\n",
|
||||
" model_inputs[\"input_ids\"][i] = torch.tensor(model_inputs[\"input_ids\"][i][:max_length])\n",
|
||||
" model_inputs[\"attention_mask\"][i] = torch.tensor(model_inputs[\"attention_mask\"][i][:max_length])\n",
|
||||
" return model_inputs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"test_dataset = dataset[\"test\"].map(\n",
|
||||
" test_preprocess_function,\n",
|
||||
" batched=True,\n",
|
||||
" num_proc=1,\n",
|
||||
" remove_columns=dataset[\"train\"].column_names,\n",
|
||||
" load_from_cache_file=False,\n",
|
||||
" desc=\"Running tokenizer on dataset\",\n",
|
||||
" )\n",
|
||||
" test_preprocess_function,\n",
|
||||
" batched=True,\n",
|
||||
" num_proc=1,\n",
|
||||
" remove_columns=dataset[\"train\"].column_names,\n",
|
||||
" load_from_cache_file=False,\n",
|
||||
" desc=\"Running tokenizer on dataset\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"test_dataloader = DataLoader(test_dataset, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True)\n",
|
||||
"next(iter(test_dataloader))"
|
||||
@@ -308,12 +312,10 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"\n",
|
||||
"# creating model\n",
|
||||
"model = AutoModelForCausalLM.from_pretrained(model_name_or_path)\n",
|
||||
"model = get_peft_model(model, peft_config)\n",
|
||||
"model.print_trainable_parameters()\n",
|
||||
"\n"
|
||||
"model.print_trainable_parameters()"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1155,8 +1157,8 @@
|
||||
" total_loss = 0\n",
|
||||
" for step, batch in enumerate(tqdm(train_dataloader)):\n",
|
||||
" batch = {k: v.to(device) for k, v in batch.items()}\n",
|
||||
"# print(batch)\n",
|
||||
"# print(batch[\"input_ids\"].shape)\n",
|
||||
" # print(batch)\n",
|
||||
" # print(batch[\"input_ids\"].shape)\n",
|
||||
" outputs = model(**batch)\n",
|
||||
" loss = outputs.loss\n",
|
||||
" total_loss += loss.detach().float()\n",
|
||||
@@ -1174,13 +1176,15 @@
|
||||
" outputs = model(**batch)\n",
|
||||
" loss = outputs.loss\n",
|
||||
" eval_loss += loss.detach().float()\n",
|
||||
" eval_preds.extend(tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True))\n",
|
||||
" eval_preds.extend(\n",
|
||||
" tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True)\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" eval_epoch_loss = eval_loss/len(train_dataloader)\n",
|
||||
" eval_epoch_loss = eval_loss / len(train_dataloader)\n",
|
||||
" eval_ppl = torch.exp(eval_epoch_loss)\n",
|
||||
" train_epoch_loss = total_loss/len(eval_dataloader)\n",
|
||||
" train_epoch_loss = total_loss / len(eval_dataloader)\n",
|
||||
" train_ppl = torch.exp(train_epoch_loss)\n",
|
||||
" print(f\"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}\")\n"
|
||||
" print(f\"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1217,10 +1221,11 @@
|
||||
"\n",
|
||||
"with torch.no_grad():\n",
|
||||
" inputs = {k: v.to(device) for k, v in inputs.items()}\n",
|
||||
" outputs = model.generate(input_ids=inputs[\"input_ids\"], attention_mask=inputs[\"attention_mask\"], max_new_tokens=10, eos_token_id=3)\n",
|
||||
" outputs = model.generate(\n",
|
||||
" input_ids=inputs[\"input_ids\"], attention_mask=inputs[\"attention_mask\"], max_new_tokens=10, eos_token_id=3\n",
|
||||
" )\n",
|
||||
" print(outputs)\n",
|
||||
" print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))\n",
|
||||
" "
|
||||
" print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1254,11 +1259,12 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from peft import PeftModel, PeftConfig\n",
|
||||
"\n",
|
||||
"peft_model_id = f\"{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}\"\n",
|
||||
"\n",
|
||||
"config = PeftConfig.from_pretrained(peft_model_id)\n",
|
||||
"model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path)\n",
|
||||
"model = PeftModel.from_pretrained(model, peft_model_id)\n"
|
||||
"model = PeftModel.from_pretrained(model, peft_model_id)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1291,10 +1297,11 @@
|
||||
"\n",
|
||||
"with torch.no_grad():\n",
|
||||
" inputs = {k: v.to(device) for k, v in inputs.items()}\n",
|
||||
" outputs = model.generate(input_ids=inputs[\"input_ids\"], attention_mask=inputs[\"attention_mask\"], max_new_tokens=10, eos_token_id=3)\n",
|
||||
" outputs = model.generate(\n",
|
||||
" input_ids=inputs[\"input_ids\"], attention_mask=inputs[\"attention_mask\"], max_new_tokens=10, eos_token_id=3\n",
|
||||
" )\n",
|
||||
" print(outputs)\n",
|
||||
" print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))\n",
|
||||
" "
|
||||
" print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
"import os\n",
|
||||
"from transformers import AutoTokenizer\n",
|
||||
"from torch.utils.data import DataLoader\n",
|
||||
"from transformers import default_data_collator,get_linear_schedule_with_warmup\n",
|
||||
"from transformers import default_data_collator, get_linear_schedule_with_warmup\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"\n",
|
||||
@@ -22,22 +22,23 @@
|
||||
"model_name_or_path = \"bigscience/bloomz-560m\"\n",
|
||||
"tokenizer_name_or_path = \"bigscience/bloomz-560m\"\n",
|
||||
"peft_config = PromptTuningConfig(\n",
|
||||
" task_type=TaskType.CAUSAL_LM,\n",
|
||||
" prompt_tuning_init=PromptTuningInit.TEXT,\n",
|
||||
" num_virtual_tokens=8,\n",
|
||||
" prompt_tuning_init_text=\"Classify if the tweet is a complaint or not:\",\n",
|
||||
" tokenizer_name_or_path=model_name_or_path,\n",
|
||||
" )\n",
|
||||
" task_type=TaskType.CAUSAL_LM,\n",
|
||||
" prompt_tuning_init=PromptTuningInit.TEXT,\n",
|
||||
" num_virtual_tokens=8,\n",
|
||||
" prompt_tuning_init_text=\"Classify if the tweet is a complaint or not:\",\n",
|
||||
" tokenizer_name_or_path=model_name_or_path,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"dataset_name = \"twitter_complaints\"\n",
|
||||
"checkpoint_name = f\"{dataset_name}_{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}_v1.pt\".replace(\"/\", \"_\")\n",
|
||||
"checkpoint_name = f\"{dataset_name}_{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}_v1.pt\".replace(\n",
|
||||
" \"/\", \"_\"\n",
|
||||
")\n",
|
||||
"text_column = \"Tweet text\"\n",
|
||||
"label_column = \"text_label\"\n",
|
||||
"max_length=64\n",
|
||||
"max_length = 64\n",
|
||||
"lr = 3e-2\n",
|
||||
"num_epochs = 50\n",
|
||||
"batch_size=8\n",
|
||||
"\n"
|
||||
"batch_size = 8"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -57,7 +58,6 @@
|
||||
" lambda x: {\"text_label\": [classes[label] for label in x[\"Label\"]]},\n",
|
||||
" batched=True,\n",
|
||||
" num_proc=1,\n",
|
||||
" \n",
|
||||
")\n",
|
||||
"print(dataset)\n",
|
||||
"dataset[\"train\"][0]"
|
||||
@@ -76,6 +76,8 @@
|
||||
" tokenizer.pad_token_id = tokenizer.eos_token_id\n",
|
||||
"target_max_length = max([len(tokenizer(class_label)[\"input_ids\"]) for class_label in classes])\n",
|
||||
"print(target_max_length)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def preprocess_function(examples):\n",
|
||||
" batch_size = len(examples[text_column])\n",
|
||||
" inputs = [f\"{text_column} : {x} Label : \" for x in examples[text_column]]\n",
|
||||
@@ -85,47 +87,45 @@
|
||||
" for i in range(batch_size):\n",
|
||||
" sample_input_ids = model_inputs[\"input_ids\"][i]\n",
|
||||
" label_input_ids = labels[\"input_ids\"][i] + [tokenizer.pad_token_id]\n",
|
||||
" #print(i, sample_input_ids, label_input_ids)\n",
|
||||
" model_inputs[\"input_ids\"][i] = sample_input_ids + label_input_ids \n",
|
||||
" # print(i, sample_input_ids, label_input_ids)\n",
|
||||
" model_inputs[\"input_ids\"][i] = sample_input_ids + label_input_ids\n",
|
||||
" labels[\"input_ids\"][i] = [-100] * len(sample_input_ids) + label_input_ids\n",
|
||||
" model_inputs[\"attention_mask\"][i] = [1] * len(model_inputs[\"input_ids\"][i])\n",
|
||||
" #print(model_inputs)\n",
|
||||
" # print(model_inputs)\n",
|
||||
" for i in range(batch_size):\n",
|
||||
" sample_input_ids = model_inputs[\"input_ids\"][i]\n",
|
||||
" label_input_ids = labels[\"input_ids\"][i]\n",
|
||||
" model_inputs[\"input_ids\"][i] = [tokenizer.pad_token_id]*(max_length-len(sample_input_ids)) + sample_input_ids\n",
|
||||
" model_inputs[\"attention_mask\"][i] = [0]*(max_length-len(sample_input_ids)) + model_inputs[\"attention_mask\"][i]\n",
|
||||
" labels[\"input_ids\"][i] = [-100]*(max_length-len(sample_input_ids)) + label_input_ids \n",
|
||||
" model_inputs[\"input_ids\"][i] = [tokenizer.pad_token_id] * (\n",
|
||||
" max_length - len(sample_input_ids)\n",
|
||||
" ) + sample_input_ids\n",
|
||||
" model_inputs[\"attention_mask\"][i] = [0] * (max_length - len(sample_input_ids)) + model_inputs[\n",
|
||||
" \"attention_mask\"\n",
|
||||
" ][i]\n",
|
||||
" labels[\"input_ids\"][i] = [-100] * (max_length - len(sample_input_ids)) + label_input_ids\n",
|
||||
" model_inputs[\"input_ids\"][i] = torch.tensor(model_inputs[\"input_ids\"][i][:max_length])\n",
|
||||
" model_inputs[\"attention_mask\"][i] = torch.tensor(model_inputs[\"attention_mask\"][i][:max_length])\n",
|
||||
" labels[\"input_ids\"][i] = torch.tensor(labels[\"input_ids\"][i][:max_length]) \n",
|
||||
" labels[\"input_ids\"][i] = torch.tensor(labels[\"input_ids\"][i][:max_length])\n",
|
||||
" model_inputs[\"labels\"] = labels[\"input_ids\"]\n",
|
||||
" return model_inputs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"processed_datasets = dataset.map(\n",
|
||||
" preprocess_function,\n",
|
||||
" batched=True,\n",
|
||||
" num_proc=1,\n",
|
||||
" remove_columns=dataset[\"train\"].column_names,\n",
|
||||
" load_from_cache_file=False,\n",
|
||||
" desc=\"Running tokenizer on dataset\",\n",
|
||||
" )\n",
|
||||
" preprocess_function,\n",
|
||||
" batched=True,\n",
|
||||
" num_proc=1,\n",
|
||||
" remove_columns=dataset[\"train\"].column_names,\n",
|
||||
" load_from_cache_file=False,\n",
|
||||
" desc=\"Running tokenizer on dataset\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"train_dataset = processed_datasets[\"train\"]\n",
|
||||
"eval_dataset = processed_datasets[\"train\"]\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"train_dataloader = DataLoader(\n",
|
||||
" train_dataset, shuffle=True, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True\n",
|
||||
" )\n",
|
||||
"eval_dataloader = DataLoader(eval_dataset, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" "
|
||||
" train_dataset, shuffle=True, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True\n",
|
||||
")\n",
|
||||
"eval_dataloader = DataLoader(eval_dataset, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -139,23 +139,28 @@
|
||||
" batch_size = len(examples[text_column])\n",
|
||||
" inputs = [f\"{text_column} : {x} Label : \" for x in examples[text_column]]\n",
|
||||
" model_inputs = tokenizer(inputs)\n",
|
||||
" #print(model_inputs)\n",
|
||||
" # print(model_inputs)\n",
|
||||
" for i in range(batch_size):\n",
|
||||
" sample_input_ids = model_inputs[\"input_ids\"][i]\n",
|
||||
" model_inputs[\"input_ids\"][i] = [tokenizer.pad_token_id]*(max_length-len(sample_input_ids)) + sample_input_ids\n",
|
||||
" model_inputs[\"attention_mask\"][i] = [0]*(max_length-len(sample_input_ids)) + model_inputs[\"attention_mask\"][i]\n",
|
||||
" model_inputs[\"input_ids\"][i] = [tokenizer.pad_token_id] * (\n",
|
||||
" max_length - len(sample_input_ids)\n",
|
||||
" ) + sample_input_ids\n",
|
||||
" model_inputs[\"attention_mask\"][i] = [0] * (max_length - len(sample_input_ids)) + model_inputs[\n",
|
||||
" \"attention_mask\"\n",
|
||||
" ][i]\n",
|
||||
" model_inputs[\"input_ids\"][i] = torch.tensor(model_inputs[\"input_ids\"][i][:max_length])\n",
|
||||
" model_inputs[\"attention_mask\"][i] = torch.tensor(model_inputs[\"attention_mask\"][i][:max_length])\n",
|
||||
" return model_inputs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"test_dataset = dataset[\"test\"].map(\n",
|
||||
" test_preprocess_function,\n",
|
||||
" batched=True,\n",
|
||||
" num_proc=1,\n",
|
||||
" remove_columns=dataset[\"train\"].column_names,\n",
|
||||
" load_from_cache_file=False,\n",
|
||||
" desc=\"Running tokenizer on dataset\",\n",
|
||||
" )\n",
|
||||
" test_preprocess_function,\n",
|
||||
" batched=True,\n",
|
||||
" num_proc=1,\n",
|
||||
" remove_columns=dataset[\"train\"].column_names,\n",
|
||||
" load_from_cache_file=False,\n",
|
||||
" desc=\"Running tokenizer on dataset\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"test_dataloader = DataLoader(test_dataset, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True)\n",
|
||||
"next(iter(test_dataloader))"
|
||||
@@ -198,12 +203,10 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\n",
|
||||
"# creating model\n",
|
||||
"model = AutoModelForCausalLM.from_pretrained(model_name_or_path)\n",
|
||||
"model = get_peft_model(model, peft_config)\n",
|
||||
"model.print_trainable_parameters()\n",
|
||||
"\n"
|
||||
"model.print_trainable_parameters()"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -996,8 +999,8 @@
|
||||
" total_loss = 0\n",
|
||||
" for step, batch in enumerate(tqdm(train_dataloader)):\n",
|
||||
" batch = {k: v.to(device) for k, v in batch.items()}\n",
|
||||
"# print(batch)\n",
|
||||
"# print(batch[\"input_ids\"].shape)\n",
|
||||
" # print(batch)\n",
|
||||
" # print(batch[\"input_ids\"].shape)\n",
|
||||
" outputs = model(**batch)\n",
|
||||
" loss = outputs.loss\n",
|
||||
" total_loss += loss.detach().float()\n",
|
||||
@@ -1015,13 +1018,15 @@
|
||||
" outputs = model(**batch)\n",
|
||||
" loss = outputs.loss\n",
|
||||
" eval_loss += loss.detach().float()\n",
|
||||
" eval_preds.extend(tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True))\n",
|
||||
" eval_preds.extend(\n",
|
||||
" tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True)\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" eval_epoch_loss = eval_loss/len(train_dataloader)\n",
|
||||
" eval_epoch_loss = eval_loss / len(train_dataloader)\n",
|
||||
" eval_ppl = torch.exp(eval_epoch_loss)\n",
|
||||
" train_epoch_loss = total_loss/len(eval_dataloader)\n",
|
||||
" train_epoch_loss = total_loss / len(eval_dataloader)\n",
|
||||
" train_ppl = torch.exp(train_epoch_loss)\n",
|
||||
" print(f\"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}\")\n"
|
||||
" print(f\"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1060,10 +1065,11 @@
|
||||
"\n",
|
||||
"with torch.no_grad():\n",
|
||||
" inputs = {k: v.to(device) for k, v in inputs.items()}\n",
|
||||
" outputs = model.generate(input_ids=inputs[\"input_ids\"], attention_mask=inputs[\"attention_mask\"], max_new_tokens=10, eos_token_id=3)\n",
|
||||
" outputs = model.generate(\n",
|
||||
" input_ids=inputs[\"input_ids\"], attention_mask=inputs[\"attention_mask\"], max_new_tokens=10, eos_token_id=3\n",
|
||||
" )\n",
|
||||
" print(outputs)\n",
|
||||
" print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))\n",
|
||||
" "
|
||||
" print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1109,11 +1115,12 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from peft import PeftModel, PeftConfig\n",
|
||||
"\n",
|
||||
"peft_model_id = f\"{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}\"\n",
|
||||
"\n",
|
||||
"config = PeftConfig.from_pretrained(peft_model_id)\n",
|
||||
"model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path)\n",
|
||||
"model = PeftModel.from_pretrained(model, peft_model_id)\n"
|
||||
"model = PeftModel.from_pretrained(model, peft_model_id)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1146,10 +1153,11 @@
|
||||
"\n",
|
||||
"with torch.no_grad():\n",
|
||||
" inputs = {k: v.to(device) for k, v in inputs.items()}\n",
|
||||
" outputs = model.generate(input_ids=inputs[\"input_ids\"], attention_mask=inputs[\"attention_mask\"], max_new_tokens=10, eos_token_id=3)\n",
|
||||
" outputs = model.generate(\n",
|
||||
" input_ids=inputs[\"input_ids\"], attention_mask=inputs[\"attention_mask\"], max_new_tokens=10, eos_token_id=3\n",
|
||||
" )\n",
|
||||
" print(outputs)\n",
|
||||
" print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))\n",
|
||||
" "
|
||||
" print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -24,14 +24,15 @@
|
||||
],
|
||||
"source": [
|
||||
"from transformers import AutoModelForSeq2SeqLM\n",
|
||||
"from peft import get_peft_config,get_peft_model, get_peft_model_state_dict, LoraConfig, TaskType\n",
|
||||
"from peft import get_peft_config, get_peft_model, get_peft_model_state_dict, LoraConfig, TaskType\n",
|
||||
"import torch\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n",
|
||||
"from transformers import AutoTokenizer\n",
|
||||
"from torch.utils.data import DataLoader\n",
|
||||
"from transformers import default_data_collator,get_linear_schedule_with_warmup\n",
|
||||
"from transformers import default_data_collator, get_linear_schedule_with_warmup\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"\n",
|
||||
@@ -42,10 +43,10 @@
|
||||
"checkpoint_name = \"financial_sentiment_analysis_lora_v1.pt\"\n",
|
||||
"text_column = \"sentence\"\n",
|
||||
"label_column = \"text_label\"\n",
|
||||
"max_length=128\n",
|
||||
"max_length = 128\n",
|
||||
"lr = 1e-3\n",
|
||||
"num_epochs = 3\n",
|
||||
"batch_size=8\n"
|
||||
"batch_size = 8"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -56,9 +57,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# creating model\n",
|
||||
"peft_config = LoraConfig(\n",
|
||||
" task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1\n",
|
||||
")\n",
|
||||
"peft_config = LoraConfig(task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1)\n",
|
||||
"\n",
|
||||
"model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path)\n",
|
||||
"model = get_peft_model(model, peft_config)\n",
|
||||
@@ -136,17 +135,16 @@
|
||||
],
|
||||
"source": [
|
||||
"# loading dataset\n",
|
||||
"dataset = load_dataset(\"financial_phrasebank\", 'sentences_allagree')\n",
|
||||
"dataset = load_dataset(\"financial_phrasebank\", \"sentences_allagree\")\n",
|
||||
"dataset = dataset[\"train\"].train_test_split(test_size=0.1)\n",
|
||||
"dataset[\"validation\"] = dataset[\"test\"]\n",
|
||||
"del(dataset[\"test\"])\n",
|
||||
"del dataset[\"test\"]\n",
|
||||
"\n",
|
||||
"classes = dataset[\"train\"].features[\"label\"].names\n",
|
||||
"dataset = dataset.map(\n",
|
||||
" lambda x: {\"text_label\": [classes[label] for label in x[\"label\"]]},\n",
|
||||
" batched=True,\n",
|
||||
" num_proc=1,\n",
|
||||
" \n",
|
||||
")\n",
|
||||
"\n",
|
||||
"dataset[\"train\"][0]"
|
||||
@@ -190,36 +188,35 @@
|
||||
"source": [
|
||||
"# data preprocessing\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def preprocess_function(examples):\n",
|
||||
" inputs = examples[text_column]\n",
|
||||
" targets = examples[label_column]\n",
|
||||
" model_inputs = tokenizer(inputs, max_length=max_length, padding=\"max_length\", truncation=True, return_tensors=\"pt\")\n",
|
||||
" labels = tokenizer(targets, max_length=3, padding=\"max_length\", truncation=True, return_tensors=\"pt\")\n",
|
||||
" labels = labels[\"input_ids\"]\n",
|
||||
" labels[labels==tokenizer.pad_token_id] = -100\n",
|
||||
" labels[labels == tokenizer.pad_token_id] = -100\n",
|
||||
" model_inputs[\"labels\"] = labels\n",
|
||||
" return model_inputs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"processed_datasets = dataset.map(\n",
|
||||
" preprocess_function,\n",
|
||||
" batched=True,\n",
|
||||
" num_proc=1,\n",
|
||||
" remove_columns=dataset[\"train\"].column_names,\n",
|
||||
" load_from_cache_file=False,\n",
|
||||
" desc=\"Running tokenizer on dataset\",\n",
|
||||
" )\n",
|
||||
" preprocess_function,\n",
|
||||
" batched=True,\n",
|
||||
" num_proc=1,\n",
|
||||
" remove_columns=dataset[\"train\"].column_names,\n",
|
||||
" load_from_cache_file=False,\n",
|
||||
" desc=\"Running tokenizer on dataset\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"train_dataset = processed_datasets[\"train\"]\n",
|
||||
"eval_dataset = processed_datasets[\"validation\"]\n",
|
||||
"\n",
|
||||
"train_dataloader = DataLoader(\n",
|
||||
" train_dataset, shuffle=True, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True\n",
|
||||
" )\n",
|
||||
"eval_dataloader = DataLoader(eval_dataset, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" "
|
||||
" train_dataset, shuffle=True, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True\n",
|
||||
")\n",
|
||||
"eval_dataloader = DataLoader(eval_dataset, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -235,7 +232,7 @@
|
||||
" optimizer=optimizer,\n",
|
||||
" num_warmup_steps=0,\n",
|
||||
" num_training_steps=(len(train_dataloader) * num_epochs),\n",
|
||||
")\n"
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -323,13 +320,15 @@
|
||||
" outputs = model(**batch)\n",
|
||||
" loss = outputs.loss\n",
|
||||
" eval_loss += loss.detach().float()\n",
|
||||
" eval_preds.extend(tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True))\n",
|
||||
" eval_preds.extend(\n",
|
||||
" tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True)\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" eval_epoch_loss = eval_loss/len(train_dataloader)\n",
|
||||
" eval_epoch_loss = eval_loss / len(train_dataloader)\n",
|
||||
" eval_ppl = torch.exp(eval_epoch_loss)\n",
|
||||
" train_epoch_loss = total_loss/len(eval_dataloader)\n",
|
||||
" train_epoch_loss = total_loss / len(eval_dataloader)\n",
|
||||
" train_ppl = torch.exp(train_epoch_loss)\n",
|
||||
" print(f\"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}\")\n"
|
||||
" print(f\"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -350,13 +349,13 @@
|
||||
],
|
||||
"source": [
|
||||
"# print accuracy\n",
|
||||
"correct =0\n",
|
||||
"correct = 0\n",
|
||||
"total = 0\n",
|
||||
"for pred,true in zip(eval_preds, dataset[\"validation\"][\"text_label\"]):\n",
|
||||
" if pred.strip()==true.strip():\n",
|
||||
" correct+=1\n",
|
||||
" total+=1 \n",
|
||||
"accuracy = correct/total*100\n",
|
||||
"for pred, true in zip(eval_preds, dataset[\"validation\"][\"text_label\"]):\n",
|
||||
" if pred.strip() == true.strip():\n",
|
||||
" correct += 1\n",
|
||||
" total += 1\n",
|
||||
"accuracy = correct / total * 100\n",
|
||||
"print(f\"{accuracy=} % on the evaluation dataset\")\n",
|
||||
"print(f\"{eval_preds[:10]=}\")\n",
|
||||
"print(f\"{dataset['validation']['text_label'][:10]=}\")"
|
||||
@@ -401,11 +400,12 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from peft import PeftModel, PeftConfig\n",
|
||||
"\n",
|
||||
"peft_model_id = f\"{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}\"\n",
|
||||
"\n",
|
||||
"config = PeftConfig.from_pretrained(peft_model_id)\n",
|
||||
"model = AutoModelForSeq2SeqLM.from_pretrained(config.base_model_name_or_path)\n",
|
||||
"model = PeftModel.from_pretrained(model, peft_model_id)\n"
|
||||
"model = PeftModel.from_pretrained(model, peft_model_id)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -437,7 +437,7 @@
|
||||
"with torch.no_grad():\n",
|
||||
" outputs = model.generate(input_ids=inputs[\"input_ids\"], max_new_tokens=10)\n",
|
||||
" print(outputs)\n",
|
||||
" print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))\n"
|
||||
" print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
+13
-15
@@ -14,17 +14,17 @@
|
||||
"import os\n",
|
||||
"from transformers import AutoTokenizer\n",
|
||||
"from torch.utils.data import DataLoader\n",
|
||||
"from transformers import default_data_collator,get_linear_schedule_with_warmup\n",
|
||||
"from transformers import default_data_collator, get_linear_schedule_with_warmup\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"\n",
|
||||
"dataset_name = \"twitter_complaints\"\n",
|
||||
"text_column = \"Tweet text\"\n",
|
||||
"label_column = \"text_label\"\n",
|
||||
"batch_size=8\n",
|
||||
"batch_size = 8\n",
|
||||
"\n",
|
||||
"peft_model_id = \"smangrul/twitter_complaints_bigscience_T0_3B_LORA_SEQ_2_SEQ_LM\"\n",
|
||||
"config = PeftConfig.from_pretrained(peft_model_id)\n"
|
||||
"config = PeftConfig.from_pretrained(peft_model_id)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -35,7 +35,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"peft_model_id = \"smangrul/twitter_complaints_bigscience_T0_3B_LORA_SEQ_2_SEQ_LM\"\n",
|
||||
"max_memory={0: \"6GIB\", 1: \"0GIB\", 2: \"0GIB\", 3: \"0GIB\", 4: \"0GIB\", \"cpu\":\"30GB\"}\n",
|
||||
"max_memory = {0: \"6GIB\", 1: \"0GIB\", 2: \"0GIB\", 3: \"0GIB\", 4: \"0GIB\", \"cpu\": \"30GB\"}\n",
|
||||
"config = PeftConfig.from_pretrained(peft_model_id)\n",
|
||||
"model = AutoModelForSeq2SeqLM.from_pretrained(config.base_model_name_or_path, device_map=\"auto\", max_memory=max_memory)\n",
|
||||
"model = PeftModel.from_pretrained(model, peft_model_id, device_map=\"auto\", max_memory=max_memory)"
|
||||
@@ -58,7 +58,6 @@
|
||||
" lambda x: {\"text_label\": [classes[label] for label in x[\"Label\"]]},\n",
|
||||
" batched=True,\n",
|
||||
" num_proc=1,\n",
|
||||
" \n",
|
||||
")\n",
|
||||
"print(dataset)\n",
|
||||
"dataset[\"train\"][0]"
|
||||
@@ -73,6 +72,8 @@
|
||||
"source": [
|
||||
"tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)\n",
|
||||
"target_max_length = max([len(tokenizer(class_label)[\"input_ids\"]) for class_label in classes])\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def preprocess_function(examples):\n",
|
||||
" inputs = examples[text_column]\n",
|
||||
" targets = examples[label_column]\n",
|
||||
@@ -85,6 +86,7 @@
|
||||
" model_inputs[\"labels\"] = labels\n",
|
||||
" return model_inputs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"processed_datasets = dataset.map(\n",
|
||||
" preprocess_function,\n",
|
||||
" batched=True,\n",
|
||||
@@ -100,18 +102,14 @@
|
||||
"\n",
|
||||
"\n",
|
||||
"def collate_fn(examples):\n",
|
||||
" return tokenizer.pad(examples, padding=\"longest\", return_tensors=\"pt\")\n",
|
||||
" return tokenizer.pad(examples, padding=\"longest\", return_tensors=\"pt\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"train_dataloader = DataLoader(\n",
|
||||
" train_dataset, shuffle=True, collate_fn=collate_fn, batch_size=batch_size, pin_memory=True\n",
|
||||
")\n",
|
||||
"eval_dataloader = DataLoader(eval_dataset, collate_fn=collate_fn, batch_size=batch_size, pin_memory=True)\n",
|
||||
"test_dataloader = DataLoader(test_dataset, collate_fn=collate_fn, batch_size=batch_size, pin_memory=True)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" "
|
||||
"test_dataloader = DataLoader(test_dataset, collate_fn=collate_fn, batch_size=batch_size, pin_memory=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -146,7 +144,7 @@
|
||||
"with torch.no_grad():\n",
|
||||
" outputs = model.generate(input_ids=inputs[\"input_ids\"].to(\"cuda\"), max_new_tokens=10)\n",
|
||||
" print(outputs)\n",
|
||||
" print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))\n"
|
||||
" print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -201,7 +199,7 @@
|
||||
"accuracy = correct / total * 100\n",
|
||||
"print(f\"{accuracy=}\")\n",
|
||||
"print(f\"{eval_preds[:10]=}\")\n",
|
||||
"print(f\"{dataset['train'][label_column][:10]=}\")\n"
|
||||
"print(f\"{dataset['train'][label_column][:10]=}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -220,7 +218,7 @@
|
||||
" outputs = model.generate(**batch, max_new_tokens=10)\n",
|
||||
" preds = outputs.detach().cpu().numpy()\n",
|
||||
" test_preds.extend(tokenizer.batch_decode(preds, skip_special_tokens=True))\n",
|
||||
" if len(test_preds)>100:\n",
|
||||
" if len(test_preds) > 100:\n",
|
||||
" break\n",
|
||||
"test_preds"
|
||||
]
|
||||
|
||||
@@ -24,15 +24,16 @@
|
||||
],
|
||||
"source": [
|
||||
"from transformers import AutoModelForSeq2SeqLM\n",
|
||||
"from peft import get_peft_config,get_peft_model, get_peft_model_state_dict, PrefixTuningConfig, TaskType\n",
|
||||
"from peft import get_peft_config, get_peft_model, get_peft_model_state_dict, PrefixTuningConfig, TaskType\n",
|
||||
"import torch\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n",
|
||||
"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"3\"\n",
|
||||
"from transformers import AutoTokenizer\n",
|
||||
"from torch.utils.data import DataLoader\n",
|
||||
"from transformers import default_data_collator,get_linear_schedule_with_warmup\n",
|
||||
"from transformers import default_data_collator, get_linear_schedule_with_warmup\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"\n",
|
||||
@@ -43,10 +44,10 @@
|
||||
"checkpoint_name = \"financial_sentiment_analysis_prefix_tuning_v1.pt\"\n",
|
||||
"text_column = \"sentence\"\n",
|
||||
"label_column = \"text_label\"\n",
|
||||
"max_length=128\n",
|
||||
"max_length = 128\n",
|
||||
"lr = 1e-2\n",
|
||||
"num_epochs = 5\n",
|
||||
"batch_size=8\n"
|
||||
"batch_size = 8"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -57,9 +58,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# creating model\n",
|
||||
"peft_config = PrefixTuningConfig(\n",
|
||||
" task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, num_virtual_tokens=20\n",
|
||||
")\n",
|
||||
"peft_config = PrefixTuningConfig(task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, num_virtual_tokens=20)\n",
|
||||
"\n",
|
||||
"model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path)\n",
|
||||
"model = get_peft_model(model, peft_config)\n",
|
||||
@@ -137,17 +136,16 @@
|
||||
],
|
||||
"source": [
|
||||
"# loading dataset\n",
|
||||
"dataset = load_dataset(\"financial_phrasebank\", 'sentences_allagree')\n",
|
||||
"dataset = load_dataset(\"financial_phrasebank\", \"sentences_allagree\")\n",
|
||||
"dataset = dataset[\"train\"].train_test_split(test_size=0.1)\n",
|
||||
"dataset[\"validation\"] = dataset[\"test\"]\n",
|
||||
"del(dataset[\"test\"])\n",
|
||||
"del dataset[\"test\"]\n",
|
||||
"\n",
|
||||
"classes = dataset[\"train\"].features[\"label\"].names\n",
|
||||
"dataset = dataset.map(\n",
|
||||
" lambda x: {\"text_label\": [classes[label] for label in x[\"label\"]]},\n",
|
||||
" batched=True,\n",
|
||||
" num_proc=1,\n",
|
||||
" \n",
|
||||
")\n",
|
||||
"\n",
|
||||
"dataset[\"train\"][0]"
|
||||
@@ -203,36 +201,35 @@
|
||||
"source": [
|
||||
"# data preprocessing\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def preprocess_function(examples):\n",
|
||||
" inputs = examples[text_column]\n",
|
||||
" targets = examples[label_column]\n",
|
||||
" model_inputs = tokenizer(inputs, max_length=max_length, padding=\"max_length\", truncation=True, return_tensors=\"pt\")\n",
|
||||
" labels = tokenizer(targets, max_length=2, padding=\"max_length\", truncation=True, return_tensors=\"pt\")\n",
|
||||
" labels = labels[\"input_ids\"]\n",
|
||||
" labels[labels==tokenizer.pad_token_id] = -100\n",
|
||||
" labels[labels == tokenizer.pad_token_id] = -100\n",
|
||||
" model_inputs[\"labels\"] = labels\n",
|
||||
" return model_inputs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"processed_datasets = dataset.map(\n",
|
||||
" preprocess_function,\n",
|
||||
" batched=True,\n",
|
||||
" num_proc=1,\n",
|
||||
" remove_columns=dataset[\"train\"].column_names,\n",
|
||||
" load_from_cache_file=False,\n",
|
||||
" desc=\"Running tokenizer on dataset\",\n",
|
||||
" )\n",
|
||||
" preprocess_function,\n",
|
||||
" batched=True,\n",
|
||||
" num_proc=1,\n",
|
||||
" remove_columns=dataset[\"train\"].column_names,\n",
|
||||
" load_from_cache_file=False,\n",
|
||||
" desc=\"Running tokenizer on dataset\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"train_dataset = processed_datasets[\"train\"]\n",
|
||||
"eval_dataset = processed_datasets[\"validation\"]\n",
|
||||
"\n",
|
||||
"train_dataloader = DataLoader(\n",
|
||||
" train_dataset, shuffle=True, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True\n",
|
||||
" )\n",
|
||||
"eval_dataloader = DataLoader(eval_dataset, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" "
|
||||
" train_dataset, shuffle=True, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True\n",
|
||||
")\n",
|
||||
"eval_dataloader = DataLoader(eval_dataset, collate_fn=default_data_collator, batch_size=batch_size, pin_memory=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -248,7 +245,7 @@
|
||||
" optimizer=optimizer,\n",
|
||||
" num_warmup_steps=0,\n",
|
||||
" num_training_steps=(len(train_dataloader) * num_epochs),\n",
|
||||
")\n"
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -359,13 +356,15 @@
|
||||
" outputs = model(**batch)\n",
|
||||
" loss = outputs.loss\n",
|
||||
" eval_loss += loss.detach().float()\n",
|
||||
" eval_preds.extend(tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True))\n",
|
||||
" eval_preds.extend(\n",
|
||||
" tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True)\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" eval_epoch_loss = eval_loss/len(train_dataloader)\n",
|
||||
" eval_epoch_loss = eval_loss / len(train_dataloader)\n",
|
||||
" eval_ppl = torch.exp(eval_epoch_loss)\n",
|
||||
" train_epoch_loss = total_loss/len(eval_dataloader)\n",
|
||||
" train_epoch_loss = total_loss / len(eval_dataloader)\n",
|
||||
" train_ppl = torch.exp(train_epoch_loss)\n",
|
||||
" print(f\"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}\")\n"
|
||||
" print(f\"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -386,13 +385,13 @@
|
||||
],
|
||||
"source": [
|
||||
"# print accuracy\n",
|
||||
"correct =0\n",
|
||||
"correct = 0\n",
|
||||
"total = 0\n",
|
||||
"for pred,true in zip(eval_preds, dataset[\"validation\"][\"text_label\"]):\n",
|
||||
" if pred.strip()==true.strip():\n",
|
||||
" correct+=1\n",
|
||||
" total+=1 \n",
|
||||
"accuracy = correct/total*100\n",
|
||||
"for pred, true in zip(eval_preds, dataset[\"validation\"][\"text_label\"]):\n",
|
||||
" if pred.strip() == true.strip():\n",
|
||||
" correct += 1\n",
|
||||
" total += 1\n",
|
||||
"accuracy = correct / total * 100\n",
|
||||
"print(f\"{accuracy=} % on the evaluation dataset\")\n",
|
||||
"print(f\"{eval_preds[:10]=}\")\n",
|
||||
"print(f\"{dataset['validation']['text_label'][:10]=}\")"
|
||||
@@ -437,11 +436,12 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from peft import PeftModel, PeftConfig\n",
|
||||
"\n",
|
||||
"peft_model_id = f\"{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}\"\n",
|
||||
"\n",
|
||||
"config = PeftConfig.from_pretrained(peft_model_id)\n",
|
||||
"model = AutoModelForSeq2SeqLM.from_pretrained(config.base_model_name_or_path)\n",
|
||||
"model = PeftModel.from_pretrained(model, peft_model_id)\n"
|
||||
"model = PeftModel.from_pretrained(model, peft_model_id)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -475,7 +475,7 @@
|
||||
"with torch.no_grad():\n",
|
||||
" outputs = model.generate(input_ids=inputs[\"input_ids\"], max_new_tokens=10)\n",
|
||||
" print(outputs)\n",
|
||||
" print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))\n"
|
||||
" print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -155,7 +155,7 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import transformers \n",
|
||||
"import transformers\n",
|
||||
"import accelerate\n",
|
||||
"import peft"
|
||||
]
|
||||
@@ -204,9 +204,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model_checkpoint = (\n",
|
||||
" \"google/vit-base-patch16-224-in21k\" # pre-trained model from which to fine-tune\n",
|
||||
")"
|
||||
"model_checkpoint = \"google/vit-base-patch16-224-in21k\" # pre-trained model from which to fine-tune"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -736,17 +734,13 @@
|
||||
"\n",
|
||||
"def preprocess_train(example_batch):\n",
|
||||
" \"\"\"Apply train_transforms across a batch.\"\"\"\n",
|
||||
" example_batch[\"pixel_values\"] = [\n",
|
||||
" train_transforms(image.convert(\"RGB\")) for image in example_batch[\"image\"]\n",
|
||||
" ]\n",
|
||||
" example_batch[\"pixel_values\"] = [train_transforms(image.convert(\"RGB\")) for image in example_batch[\"image\"]]\n",
|
||||
" return example_batch\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def preprocess_val(example_batch):\n",
|
||||
" \"\"\"Apply val_transforms across a batch.\"\"\"\n",
|
||||
" example_batch[\"pixel_values\"] = [\n",
|
||||
" val_transforms(image.convert(\"RGB\")) for image in example_batch[\"image\"]\n",
|
||||
" ]\n",
|
||||
" example_batch[\"pixel_values\"] = [val_transforms(image.convert(\"RGB\")) for image in example_batch[\"image\"]]\n",
|
||||
" return example_batch"
|
||||
]
|
||||
},
|
||||
@@ -1099,6 +1093,7 @@
|
||||
"\n",
|
||||
"metric = evaluate.load(\"accuracy\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# the compute_metrics function takes a Named Tuple as input:\n",
|
||||
"# predictions, which are the logits of the model as Numpy arrays,\n",
|
||||
"# and label_ids, which are the ground-truth labels as Numpy arrays.\n",
|
||||
@@ -1129,6 +1124,7 @@
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def collate_fn(examples):\n",
|
||||
" pixel_values = torch.stack([example[\"pixel_values\"] for example in examples])\n",
|
||||
" labels = torch.tensor([example[\"label\"] for example in examples])\n",
|
||||
@@ -2230,10 +2226,10 @@
|
||||
"\n",
|
||||
"config = PeftConfig.from_pretrained(repo_name)\n",
|
||||
"model = model = AutoModelForImageClassification.from_pretrained(\n",
|
||||
" config.base_model_name_or_path, \n",
|
||||
" config.base_model_name_or_path,\n",
|
||||
" label2id=label2id,\n",
|
||||
" id2label=id2label,\n",
|
||||
" ignore_mismatched_sizes=True, # provide this in case you're planning to fine-tune an already fine-tuned checkpoint\n",
|
||||
" ignore_mismatched_sizes=True, # provide this in case you're planning to fine-tune an already fine-tuned checkpoint\n",
|
||||
")\n",
|
||||
"# Load the Lora model\n",
|
||||
"inference_model = PeftModel.from_pretrained(model, repo_name)"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -51,7 +51,7 @@
|
||||
"logger = get_logger(__name__)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"MODEL_NAME=\"CompVis/stable-diffusion-v1-4\"#\"stabilityai/stable-diffusion-2-1-base\"\n",
|
||||
"MODEL_NAME = \"CompVis/stable-diffusion-v1-4\" # \"stabilityai/stable-diffusion-2-1-base\"\n",
|
||||
"INSTANCE_PROMPT = \"a photo of sks dog\"\n",
|
||||
"ckpt_dir = \"/home/sourab/temp/sd_dog_dreambooth/\""
|
||||
]
|
||||
@@ -89,31 +89,32 @@
|
||||
" with open(f\"{ckpt_dir}{instance_prompt}_lora_config.json\", \"r\") as f:\n",
|
||||
" lora_config = json.load(f)\n",
|
||||
" print(lora_config)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" checkpoint = f\"{ckpt_dir}{instance_prompt}_lora.pt\"\n",
|
||||
" lora_checkpoint_sd = torch.load(checkpoint)\n",
|
||||
" unet_lora_ds = {k:v for k,v in lora_checkpoint_sd.items() if \"text_encoder_\" not in k}\n",
|
||||
" text_encoder_lora_ds = {k.replace(\"text_encoder_\", \"\"):v for k,v in lora_checkpoint_sd.items() if \"text_encoder_\" in k}\n",
|
||||
" \n",
|
||||
" unet_lora_ds = {k: v for k, v in lora_checkpoint_sd.items() if \"text_encoder_\" not in k}\n",
|
||||
" text_encoder_lora_ds = {\n",
|
||||
" k.replace(\"text_encoder_\", \"\"): v for k, v in lora_checkpoint_sd.items() if \"text_encoder_\" in k\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" unet_config = LoraConfig(**lora_config[\"peft_config\"])\n",
|
||||
" pipe.unet = LoraModel(unet_config, pipe.unet)\n",
|
||||
" set_peft_model_state_dict(pipe.unet, unet_lora_ds) \n",
|
||||
" \n",
|
||||
" set_peft_model_state_dict(pipe.unet, unet_lora_ds)\n",
|
||||
"\n",
|
||||
" if \"text_encoder_peft_config\" in lora_config:\n",
|
||||
" text_encoder_config = LoraConfig(**lora_config[\"text_encoder_peft_config\"])\n",
|
||||
" pipe.text_encoder = LoraModel(text_encoder_config, pipe.text_encoder)\n",
|
||||
" set_peft_model_state_dict(pipe.text_encoder, text_encoder_lora_ds)\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" if dtype in (torch.float16, torch.bfloat16):\n",
|
||||
" pipe.unet.half()\n",
|
||||
" pipe.text_encoder.half()\n",
|
||||
" \n",
|
||||
" pipe.to(device) \n",
|
||||
" return pipe\n",
|
||||
" \n",
|
||||
"pipe = load_and_set_lora_ckpt(pipe, ckpt_dir, INSTANCE_PROMPT, \"cuda\", torch.float16)\n",
|
||||
"\n",
|
||||
" "
|
||||
" pipe.to(device)\n",
|
||||
" return pipe\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"pipe = load_and_set_lora_ckpt(pipe, ckpt_dir, INSTANCE_PROMPT, \"cuda\", torch.float16)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -175,9 +176,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"prompt = \"sks dog with Eiffel Tower in the background\"\n",
|
||||
"image = pipe(prompt, num_inference_steps=50, \n",
|
||||
" guidance_scale=7.5, \n",
|
||||
" negative_prompt=negative_prompt).images[0]\n",
|
||||
"image = pipe(prompt, num_inference_steps=50, guidance_scale=7.5, negative_prompt=negative_prompt).images[0]\n",
|
||||
"image"
|
||||
]
|
||||
},
|
||||
|
||||
@@ -170,9 +170,7 @@
|
||||
"\n",
|
||||
"repo_id = \"huggingface/label-files\"\n",
|
||||
"filename = \"ade20k-id2label.json\"\n",
|
||||
"id2label = json.load(\n",
|
||||
" open(cached_download(hf_hub_url(repo_id, filename, repo_type=\"dataset\")), \"r\")\n",
|
||||
")\n",
|
||||
"id2label = json.load(open(cached_download(hf_hub_url(repo_id, filename, repo_type=\"dataset\")), \"r\"))\n",
|
||||
"id2label = {int(k): v for k, v in id2label.items()}\n",
|
||||
"label2id = {v: k for k, v in id2label.items()}\n",
|
||||
"num_labels = len(id2label)"
|
||||
@@ -318,12 +316,8 @@
|
||||
" per_category_accuracy = metrics.pop(\"per_category_accuracy\").tolist()\n",
|
||||
" per_category_iou = metrics.pop(\"per_category_iou\").tolist()\n",
|
||||
"\n",
|
||||
" metrics.update(\n",
|
||||
" {f\"accuracy_{id2label[i]}\": v for i, v in enumerate(per_category_accuracy)}\n",
|
||||
" )\n",
|
||||
" metrics.update(\n",
|
||||
" {f\"iou_{id2label[i]}\": v for i, v in enumerate(per_category_iou)}\n",
|
||||
" )\n",
|
||||
" metrics.update({f\"accuracy_{id2label[i]}\": v for i, v in enumerate(per_category_accuracy)})\n",
|
||||
" metrics.update({f\"iou_{id2label[i]}\": v for i, v in enumerate(per_category_iou)})\n",
|
||||
"\n",
|
||||
" return metrics"
|
||||
]
|
||||
@@ -1022,9 +1016,7 @@
|
||||
" color_seg[pred_seg == label, :] = color\n",
|
||||
"color_seg = color_seg[..., ::-1] # convert to BGR\n",
|
||||
"\n",
|
||||
"img = (\n",
|
||||
" np.array(image) * 0.5 + color_seg * 0.5\n",
|
||||
") # plot the image with the segmentation map\n",
|
||||
"img = np.array(image) * 0.5 + color_seg * 0.5 # plot the image with the segmentation map\n",
|
||||
"img = img.astype(np.uint8)\n",
|
||||
"\n",
|
||||
"plt.figure(figsize=(15, 10))\n",
|
||||
|
||||
@@ -29,13 +29,21 @@
|
||||
"import torch\n",
|
||||
"from torch.optim import AdamW\n",
|
||||
"from torch.utils.data import DataLoader\n",
|
||||
"from peft import get_peft_config,get_peft_model, get_peft_model_state_dict, set_peft_model_state_dict, LoraConfig, PeftType, \\\n",
|
||||
"PrefixTuningConfig, PromptEncoderConfig\n",
|
||||
"from peft import (\n",
|
||||
" get_peft_config,\n",
|
||||
" get_peft_model,\n",
|
||||
" get_peft_model_state_dict,\n",
|
||||
" set_peft_model_state_dict,\n",
|
||||
" LoraConfig,\n",
|
||||
" PeftType,\n",
|
||||
" PrefixTuningConfig,\n",
|
||||
" PromptEncoderConfig,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"import evaluate\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed\n",
|
||||
"from tqdm import tqdm\n"
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -60,13 +68,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"peft_config = LoraConfig(\n",
|
||||
" task_type=\"SEQ_CLS\",\n",
|
||||
" inference_mode=False,\n",
|
||||
" r=8,\n",
|
||||
" lora_alpha=16,\n",
|
||||
" lora_dropout=0.1\n",
|
||||
")\n",
|
||||
"peft_config = LoraConfig(task_type=\"SEQ_CLS\", inference_mode=False, r=8, lora_alpha=16, lora_dropout=0.1)\n",
|
||||
"lr = 3e-4"
|
||||
]
|
||||
},
|
||||
@@ -159,19 +161,21 @@
|
||||
" padding_side = \"left\"\n",
|
||||
"else:\n",
|
||||
" padding_side = \"right\"\n",
|
||||
" \n",
|
||||
"\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, padding_side=padding_side)\n",
|
||||
"if getattr(tokenizer, \"pad_token_id\") is None:\n",
|
||||
" tokenizer.pad_token_id = tokenizer.eos_token_id\n",
|
||||
" \n",
|
||||
"\n",
|
||||
"datasets = load_dataset(\"glue\", task)\n",
|
||||
"metric = evaluate.load(\"glue\", task)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def tokenize_function(examples):\n",
|
||||
" # max_length=None => use the model max length (it's actually the default)\n",
|
||||
" outputs = tokenizer(examples[\"sentence1\"], examples[\"sentence2\"], truncation=True, max_length=None)\n",
|
||||
" return outputs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"tokenized_datasets = datasets.map(\n",
|
||||
" tokenize_function,\n",
|
||||
" batched=True,\n",
|
||||
@@ -182,16 +186,16 @@
|
||||
"# transformers library\n",
|
||||
"tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def collate_fn(examples):\n",
|
||||
" return tokenizer.pad(examples, padding=\"longest\", return_tensors=\"pt\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Instantiate dataloaders.\n",
|
||||
"train_dataloader = DataLoader(\n",
|
||||
" tokenized_datasets[\"train\"], shuffle=True, collate_fn=collate_fn, batch_size=batch_size\n",
|
||||
")\n",
|
||||
"train_dataloader = DataLoader(tokenized_datasets[\"train\"], shuffle=True, collate_fn=collate_fn, batch_size=batch_size)\n",
|
||||
"eval_dataloader = DataLoader(\n",
|
||||
" tokenized_datasets[\"validation\"], shuffle=False, collate_fn=collate_fn, batch_size=batch_size\n",
|
||||
")\n"
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -219,7 +223,7 @@
|
||||
"# Instantiate scheduler\n",
|
||||
"lr_scheduler = get_linear_schedule_with_warmup(\n",
|
||||
" optimizer=optimizer,\n",
|
||||
" num_warmup_steps=0.06*(len(train_dataloader) * num_epochs),\n",
|
||||
" num_warmup_steps=0.06 * (len(train_dataloader) * num_epochs),\n",
|
||||
" num_training_steps=(len(train_dataloader) * num_epochs),\n",
|
||||
")"
|
||||
]
|
||||
@@ -668,7 +672,7 @@
|
||||
" )\n",
|
||||
"\n",
|
||||
"eval_metric = metric.compute()\n",
|
||||
"print(eval_metric)\n"
|
||||
"print(eval_metric)"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -29,13 +29,20 @@
|
||||
"import torch\n",
|
||||
"from torch.optim import AdamW\n",
|
||||
"from torch.utils.data import DataLoader\n",
|
||||
"from peft import get_peft_config,get_peft_model, get_peft_model_state_dict, set_peft_model_state_dict, PeftType, \\\n",
|
||||
"PrefixTuningConfig, PromptEncoderConfig\n",
|
||||
"from peft import (\n",
|
||||
" get_peft_config,\n",
|
||||
" get_peft_model,\n",
|
||||
" get_peft_model_state_dict,\n",
|
||||
" set_peft_model_state_dict,\n",
|
||||
" PeftType,\n",
|
||||
" PrefixTuningConfig,\n",
|
||||
" PromptEncoderConfig,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"import evaluate\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed\n",
|
||||
"from tqdm import tqdm\n"
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -60,12 +67,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"\n",
|
||||
"peft_config = PromptEncoderConfig(\n",
|
||||
" task_type=\"SEQ_CLS\",\n",
|
||||
" num_virtual_tokens=20,\n",
|
||||
" encoder_hidden_size=128\n",
|
||||
")\n",
|
||||
"peft_config = PromptEncoderConfig(task_type=\"SEQ_CLS\", num_virtual_tokens=20, encoder_hidden_size=128)\n",
|
||||
"lr = 1e-3"
|
||||
]
|
||||
},
|
||||
@@ -111,19 +113,21 @@
|
||||
" padding_side = \"left\"\n",
|
||||
"else:\n",
|
||||
" padding_side = \"right\"\n",
|
||||
" \n",
|
||||
"\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, padding_side=padding_side)\n",
|
||||
"if getattr(tokenizer, \"pad_token_id\") is None:\n",
|
||||
" tokenizer.pad_token_id = tokenizer.eos_token_id\n",
|
||||
" \n",
|
||||
"\n",
|
||||
"datasets = load_dataset(\"glue\", task)\n",
|
||||
"metric = evaluate.load(\"glue\", task)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def tokenize_function(examples):\n",
|
||||
" # max_length=None => use the model max length (it's actually the default)\n",
|
||||
" outputs = tokenizer(examples[\"sentence1\"], examples[\"sentence2\"], truncation=True, max_length=None)\n",
|
||||
" return outputs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"tokenized_datasets = datasets.map(\n",
|
||||
" tokenize_function,\n",
|
||||
" batched=True,\n",
|
||||
@@ -134,16 +138,16 @@
|
||||
"# transformers library\n",
|
||||
"tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def collate_fn(examples):\n",
|
||||
" return tokenizer.pad(examples, padding=\"longest\", return_tensors=\"pt\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Instantiate dataloaders.\n",
|
||||
"train_dataloader = DataLoader(\n",
|
||||
" tokenized_datasets[\"train\"], shuffle=True, collate_fn=collate_fn, batch_size=batch_size\n",
|
||||
")\n",
|
||||
"train_dataloader = DataLoader(tokenized_datasets[\"train\"], shuffle=True, collate_fn=collate_fn, batch_size=batch_size)\n",
|
||||
"eval_dataloader = DataLoader(\n",
|
||||
" tokenized_datasets[\"validation\"], shuffle=False, collate_fn=collate_fn, batch_size=batch_size\n",
|
||||
")\n"
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -171,7 +175,7 @@
|
||||
"# Instantiate scheduler\n",
|
||||
"lr_scheduler = get_linear_schedule_with_warmup(\n",
|
||||
" optimizer=optimizer,\n",
|
||||
" num_warmup_steps=0,#0.06*(len(train_dataloader) * num_epochs),\n",
|
||||
" num_warmup_steps=0, # 0.06*(len(train_dataloader) * num_epochs),\n",
|
||||
" num_training_steps=(len(train_dataloader) * num_epochs),\n",
|
||||
")"
|
||||
]
|
||||
@@ -640,7 +644,7 @@
|
||||
" )\n",
|
||||
"\n",
|
||||
"eval_metric = metric.compute()\n",
|
||||
"print(eval_metric)\n"
|
||||
"print(eval_metric)"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -29,13 +29,21 @@
|
||||
"import torch\n",
|
||||
"from torch.optim import AdamW\n",
|
||||
"from torch.utils.data import DataLoader\n",
|
||||
"from peft import get_peft_config,get_peft_model, get_peft_model_state_dict, set_peft_model_state_dict, PeftType, \\\n",
|
||||
"PrefixTuningConfig, PromptEncoderConfig, PromptTuningConfig\n",
|
||||
"from peft import (\n",
|
||||
" get_peft_config,\n",
|
||||
" get_peft_model,\n",
|
||||
" get_peft_model_state_dict,\n",
|
||||
" set_peft_model_state_dict,\n",
|
||||
" PeftType,\n",
|
||||
" PrefixTuningConfig,\n",
|
||||
" PromptEncoderConfig,\n",
|
||||
" PromptTuningConfig,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"import evaluate\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed\n",
|
||||
"from tqdm import tqdm\n"
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -60,11 +68,8 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"peft_config = PromptTuningConfig(\n",
|
||||
" task_type=\"SEQ_CLS\",\n",
|
||||
" num_virtual_tokens=10\n",
|
||||
")\n",
|
||||
"lr = 1e-3\n"
|
||||
"peft_config = PromptTuningConfig(task_type=\"SEQ_CLS\", num_virtual_tokens=10)\n",
|
||||
"lr = 1e-3"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -109,19 +114,21 @@
|
||||
" padding_side = \"left\"\n",
|
||||
"else:\n",
|
||||
" padding_side = \"right\"\n",
|
||||
" \n",
|
||||
"\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, padding_side=padding_side)\n",
|
||||
"if getattr(tokenizer, \"pad_token_id\") is None:\n",
|
||||
" tokenizer.pad_token_id = tokenizer.eos_token_id\n",
|
||||
" \n",
|
||||
"\n",
|
||||
"datasets = load_dataset(\"glue\", task)\n",
|
||||
"metric = evaluate.load(\"glue\", task)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def tokenize_function(examples):\n",
|
||||
" # max_length=None => use the model max length (it's actually the default)\n",
|
||||
" outputs = tokenizer(examples[\"sentence1\"], examples[\"sentence2\"], truncation=True, max_length=None)\n",
|
||||
" return outputs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"tokenized_datasets = datasets.map(\n",
|
||||
" tokenize_function,\n",
|
||||
" batched=True,\n",
|
||||
@@ -132,16 +139,16 @@
|
||||
"# transformers library\n",
|
||||
"tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def collate_fn(examples):\n",
|
||||
" return tokenizer.pad(examples, padding=\"longest\", return_tensors=\"pt\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Instantiate dataloaders.\n",
|
||||
"train_dataloader = DataLoader(\n",
|
||||
" tokenized_datasets[\"train\"], shuffle=True, collate_fn=collate_fn, batch_size=batch_size\n",
|
||||
")\n",
|
||||
"train_dataloader = DataLoader(tokenized_datasets[\"train\"], shuffle=True, collate_fn=collate_fn, batch_size=batch_size)\n",
|
||||
"eval_dataloader = DataLoader(\n",
|
||||
" tokenized_datasets[\"validation\"], shuffle=False, collate_fn=collate_fn, batch_size=batch_size\n",
|
||||
")\n"
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -169,7 +176,7 @@
|
||||
"# Instantiate scheduler\n",
|
||||
"lr_scheduler = get_linear_schedule_with_warmup(\n",
|
||||
" optimizer=optimizer,\n",
|
||||
" num_warmup_steps=0.06*(len(train_dataloader) * num_epochs),\n",
|
||||
" num_warmup_steps=0.06 * (len(train_dataloader) * num_epochs),\n",
|
||||
" num_training_steps=(len(train_dataloader) * num_epochs),\n",
|
||||
")"
|
||||
]
|
||||
@@ -652,7 +659,7 @@
|
||||
" )\n",
|
||||
"\n",
|
||||
"eval_metric = metric.compute()\n",
|
||||
"print(eval_metric)\n"
|
||||
"print(eval_metric)"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -29,13 +29,20 @@
|
||||
"import torch\n",
|
||||
"from torch.optim import AdamW\n",
|
||||
"from torch.utils.data import DataLoader\n",
|
||||
"from peft import get_peft_config,get_peft_model, get_peft_model_state_dict, set_peft_model_state_dict, PeftType, \\\n",
|
||||
"PrefixTuningConfig, PromptEncoderConfig\n",
|
||||
"from peft import (\n",
|
||||
" get_peft_config,\n",
|
||||
" get_peft_model,\n",
|
||||
" get_peft_model_state_dict,\n",
|
||||
" set_peft_model_state_dict,\n",
|
||||
" PeftType,\n",
|
||||
" PrefixTuningConfig,\n",
|
||||
" PromptEncoderConfig,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"import evaluate\n",
|
||||
"from datasets import load_dataset\n",
|
||||
"from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed\n",
|
||||
"from tqdm import tqdm\n"
|
||||
"from tqdm import tqdm"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -60,10 +67,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"peft_config = PrefixTuningConfig(\n",
|
||||
" task_type=\"SEQ_CLS\",\n",
|
||||
" num_virtual_tokens=20\n",
|
||||
")\n",
|
||||
"peft_config = PrefixTuningConfig(task_type=\"SEQ_CLS\", num_virtual_tokens=20)\n",
|
||||
"lr = 1e-2"
|
||||
]
|
||||
},
|
||||
@@ -128,19 +132,21 @@
|
||||
" padding_side = \"left\"\n",
|
||||
"else:\n",
|
||||
" padding_side = \"right\"\n",
|
||||
" \n",
|
||||
"\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, padding_side=padding_side)\n",
|
||||
"if getattr(tokenizer, \"pad_token_id\") is None:\n",
|
||||
" tokenizer.pad_token_id = tokenizer.eos_token_id\n",
|
||||
" \n",
|
||||
"\n",
|
||||
"datasets = load_dataset(\"glue\", task)\n",
|
||||
"metric = evaluate.load(\"glue\", task)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def tokenize_function(examples):\n",
|
||||
" # max_length=None => use the model max length (it's actually the default)\n",
|
||||
" outputs = tokenizer(examples[\"sentence1\"], examples[\"sentence2\"], truncation=True, max_length=None)\n",
|
||||
" return outputs\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"tokenized_datasets = datasets.map(\n",
|
||||
" tokenize_function,\n",
|
||||
" batched=True,\n",
|
||||
@@ -151,16 +157,16 @@
|
||||
"# transformers library\n",
|
||||
"tokenized_datasets = tokenized_datasets.rename_column(\"label\", \"labels\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def collate_fn(examples):\n",
|
||||
" return tokenizer.pad(examples, padding=\"longest\", return_tensors=\"pt\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Instantiate dataloaders.\n",
|
||||
"train_dataloader = DataLoader(\n",
|
||||
" tokenized_datasets[\"train\"], shuffle=True, collate_fn=collate_fn, batch_size=batch_size\n",
|
||||
")\n",
|
||||
"train_dataloader = DataLoader(tokenized_datasets[\"train\"], shuffle=True, collate_fn=collate_fn, batch_size=batch_size)\n",
|
||||
"eval_dataloader = DataLoader(\n",
|
||||
" tokenized_datasets[\"validation\"], shuffle=False, collate_fn=collate_fn, batch_size=batch_size\n",
|
||||
")\n"
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -188,7 +194,7 @@
|
||||
"# Instantiate scheduler\n",
|
||||
"lr_scheduler = get_linear_schedule_with_warmup(\n",
|
||||
" optimizer=optimizer,\n",
|
||||
" num_warmup_steps=0.06*(len(train_dataloader) * num_epochs),\n",
|
||||
" num_warmup_steps=0.06 * (len(train_dataloader) * num_epochs),\n",
|
||||
" num_training_steps=(len(train_dataloader) * num_epochs),\n",
|
||||
")"
|
||||
]
|
||||
@@ -671,7 +677,7 @@
|
||||
" )\n",
|
||||
"\n",
|
||||
"eval_metric = metric.compute()\n",
|
||||
"print(eval_metric)\n"
|
||||
"print(eval_metric)"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -104,6 +104,7 @@
|
||||
"source": [
|
||||
"from PIL import Image, ImageDraw, ImageFont\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"base_path = \"/home/sourab/temp/data/dataset\"\n",
|
||||
"\n",
|
||||
"image = Image.open(os.path.join(base_path, \"training_data/images/0000971160.png\"))\n",
|
||||
@@ -135,11 +136,11 @@
|
||||
"source": [
|
||||
"import json\n",
|
||||
"\n",
|
||||
"with open(os.path.join(base_path, 'training_data/annotations/0000971160.json')) as f:\n",
|
||||
" data = json.load(f)\n",
|
||||
"with open(os.path.join(base_path, \"training_data/annotations/0000971160.json\")) as f:\n",
|
||||
" data = json.load(f)\n",
|
||||
"\n",
|
||||
"for annotation in data['form']:\n",
|
||||
" print(annotation)"
|
||||
"for annotation in data[\"form\"]:\n",
|
||||
" print(annotation)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -168,17 +169,17 @@
|
||||
"\n",
|
||||
"font = ImageFont.load_default()\n",
|
||||
"\n",
|
||||
"label2color = {'question':'blue', 'answer':'green', 'header':'orange', 'other':'violet'}\n",
|
||||
"label2color = {\"question\": \"blue\", \"answer\": \"green\", \"header\": \"orange\", \"other\": \"violet\"}\n",
|
||||
"\n",
|
||||
"for annotation in data['form']:\n",
|
||||
" label = annotation['label']\n",
|
||||
" general_box = annotation['box']\n",
|
||||
" draw.rectangle(general_box, outline=label2color[label], width=2)\n",
|
||||
" draw.text((general_box[0] + 10, general_box[1] - 10), label, fill=label2color[label], font=font)\n",
|
||||
" words = annotation['words']\n",
|
||||
" for word in words:\n",
|
||||
" box = word['box']\n",
|
||||
" draw.rectangle(box, outline=label2color[label], width=1)\n",
|
||||
"for annotation in data[\"form\"]:\n",
|
||||
" label = annotation[\"label\"]\n",
|
||||
" general_box = annotation[\"box\"]\n",
|
||||
" draw.rectangle(general_box, outline=label2color[label], width=2)\n",
|
||||
" draw.text((general_box[0] + 10, general_box[1] - 10), label, fill=label2color[label], font=font)\n",
|
||||
" words = annotation[\"words\"]\n",
|
||||
" for word in words:\n",
|
||||
" box = word[\"box\"]\n",
|
||||
" draw.rectangle(box, outline=label2color[label], width=1)\n",
|
||||
"\n",
|
||||
"image"
|
||||
]
|
||||
@@ -260,6 +261,7 @@
|
||||
"source": [
|
||||
"from torch.nn import CrossEntropyLoss\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def get_labels(path):\n",
|
||||
" with open(path, \"r\") as f:\n",
|
||||
" labels = f.read().splitlines()\n",
|
||||
@@ -267,6 +269,7 @@
|
||||
" labels = [\"O\"] + labels\n",
|
||||
" return labels\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"labels = get_labels(\"data/labels.txt\")\n",
|
||||
"num_labels = len(labels)\n",
|
||||
"label_map = {i: label for i, label in enumerate(labels)}\n",
|
||||
@@ -368,27 +371,19 @@
|
||||
" pad_token_segment_id=4 if args.model_type in [\"xlnet\"] else 0,\n",
|
||||
" pad_token_label_id=pad_token_label_id,\n",
|
||||
" )\n",
|
||||
" #if args.local_rank in [-1, 0]:\n",
|
||||
" #logger.info(\"Saving features into cached file %s\", cached_features_file)\n",
|
||||
" #torch.save(features, cached_features_file)\n",
|
||||
" # if args.local_rank in [-1, 0]:\n",
|
||||
" # logger.info(\"Saving features into cached file %s\", cached_features_file)\n",
|
||||
" # torch.save(features, cached_features_file)\n",
|
||||
"\n",
|
||||
" if args.local_rank == 0 and mode == \"train\":\n",
|
||||
" torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache\n",
|
||||
"\n",
|
||||
" self.features = features\n",
|
||||
" # Convert to Tensors and build dataset\n",
|
||||
" self.all_input_ids = torch.tensor(\n",
|
||||
" [f.input_ids for f in features], dtype=torch.long\n",
|
||||
" )\n",
|
||||
" self.all_input_mask = torch.tensor(\n",
|
||||
" [f.input_mask for f in features], dtype=torch.long\n",
|
||||
" )\n",
|
||||
" self.all_segment_ids = torch.tensor(\n",
|
||||
" [f.segment_ids for f in features], dtype=torch.long\n",
|
||||
" )\n",
|
||||
" self.all_label_ids = torch.tensor(\n",
|
||||
" [f.label_ids for f in features], dtype=torch.long\n",
|
||||
" )\n",
|
||||
" self.all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)\n",
|
||||
" self.all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)\n",
|
||||
" self.all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)\n",
|
||||
" self.all_label_ids = torch.tensor([f.label_ids for f in features], dtype=torch.long)\n",
|
||||
" self.all_bboxes = torch.tensor([f.boxes for f in features], dtype=torch.long)\n",
|
||||
"\n",
|
||||
" def __len__(self):\n",
|
||||
@@ -441,9 +436,7 @@
|
||||
" ):\n",
|
||||
" assert (\n",
|
||||
" 0 <= all(boxes) <= 1000\n",
|
||||
" ), \"Error with input bbox ({}): the coordinate value is not between 0 and 1000\".format(\n",
|
||||
" boxes\n",
|
||||
" )\n",
|
||||
" ), \"Error with input bbox ({}): the coordinate value is not between 0 and 1000\".format(boxes)\n",
|
||||
" self.input_ids = input_ids\n",
|
||||
" self.input_mask = input_mask\n",
|
||||
" self.segment_ids = segment_ids\n",
|
||||
@@ -460,9 +453,9 @@
|
||||
" image_file_path = os.path.join(data_dir, \"{}_image.txt\".format(mode))\n",
|
||||
" guid_index = 1\n",
|
||||
" examples = []\n",
|
||||
" with open(file_path, encoding=\"utf-8\") as f, open(\n",
|
||||
" box_file_path, encoding=\"utf-8\"\n",
|
||||
" ) as fb, open(image_file_path, encoding=\"utf-8\") as fi:\n",
|
||||
" with open(file_path, encoding=\"utf-8\") as f, open(box_file_path, encoding=\"utf-8\") as fb, open(\n",
|
||||
" image_file_path, encoding=\"utf-8\"\n",
|
||||
" ) as fi:\n",
|
||||
" words = []\n",
|
||||
" boxes = []\n",
|
||||
" actual_bboxes = []\n",
|
||||
@@ -546,17 +539,17 @@
|
||||
" sequence_a_segment_id=0,\n",
|
||||
" mask_padding_with_zero=True,\n",
|
||||
"):\n",
|
||||
" \"\"\" Loads a data file into a list of `InputBatch`s\n",
|
||||
" `cls_token_at_end` define the location of the CLS token:\n",
|
||||
" - False (Default, BERT/XLM pattern): [CLS] + A + [SEP] + B + [SEP]\n",
|
||||
" - True (XLNet/GPT pattern): A + [SEP] + B + [SEP] + [CLS]\n",
|
||||
" `cls_token_segment_id` define the segment id associated to the CLS token (0 for BERT, 2 for XLNet)\n",
|
||||
" \"\"\"Loads a data file into a list of `InputBatch`s\n",
|
||||
" `cls_token_at_end` define the location of the CLS token:\n",
|
||||
" - False (Default, BERT/XLM pattern): [CLS] + A + [SEP] + B + [SEP]\n",
|
||||
" - True (XLNet/GPT pattern): A + [SEP] + B + [SEP] + [CLS]\n",
|
||||
" `cls_token_segment_id` define the segment id associated to the CLS token (0 for BERT, 2 for XLNet)\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" label_map = {label: i for i, label in enumerate(label_list)}\n",
|
||||
"\n",
|
||||
" features = []\n",
|
||||
" for (ex_index, example) in enumerate(examples):\n",
|
||||
" for ex_index, example in enumerate(examples):\n",
|
||||
" file_name = example.file_name\n",
|
||||
" page_size = example.page_size\n",
|
||||
" width, height = page_size\n",
|
||||
@@ -567,17 +560,13 @@
|
||||
" token_boxes = []\n",
|
||||
" actual_bboxes = []\n",
|
||||
" label_ids = []\n",
|
||||
" for word, label, box, actual_bbox in zip(\n",
|
||||
" example.words, example.labels, example.boxes, example.actual_bboxes\n",
|
||||
" ):\n",
|
||||
" for word, label, box, actual_bbox in zip(example.words, example.labels, example.boxes, example.actual_bboxes):\n",
|
||||
" word_tokens = tokenizer.tokenize(word)\n",
|
||||
" tokens.extend(word_tokens)\n",
|
||||
" token_boxes.extend([box] * len(word_tokens))\n",
|
||||
" actual_bboxes.extend([actual_bbox] * len(word_tokens))\n",
|
||||
" # Use the real label id for the first token of the word, and padding ids for the remaining tokens\n",
|
||||
" label_ids.extend(\n",
|
||||
" [label_map[label]] + [pad_token_label_id] * (len(word_tokens) - 1)\n",
|
||||
" )\n",
|
||||
" label_ids.extend([label_map[label]] + [pad_token_label_id] * (len(word_tokens) - 1))\n",
|
||||
"\n",
|
||||
" # Account for [CLS] and [SEP] with \"- 2\" and with \"- 3\" for RoBERTa.\n",
|
||||
" special_tokens_count = 3 if sep_token_extra else 2\n",
|
||||
@@ -640,9 +629,7 @@
|
||||
" padding_length = max_seq_length - len(input_ids)\n",
|
||||
" if pad_on_left:\n",
|
||||
" input_ids = ([pad_token] * padding_length) + input_ids\n",
|
||||
" input_mask = (\n",
|
||||
" [0 if mask_padding_with_zero else 1] * padding_length\n",
|
||||
" ) + input_mask\n",
|
||||
" input_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + input_mask\n",
|
||||
" segment_ids = ([pad_token_segment_id] * padding_length) + segment_ids\n",
|
||||
" label_ids = ([pad_token_label_id] * padding_length) + label_ids\n",
|
||||
" token_boxes = ([pad_token_box] * padding_length) + token_boxes\n",
|
||||
@@ -682,7 +669,7 @@
|
||||
" page_size=page_size,\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" return features\n"
|
||||
" return features"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -694,16 +681,20 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from transformers import LayoutLMTokenizer\n",
|
||||
"#from .unilm.layoutlm.data.funsd import FunsdDataset, InputFeatures\n",
|
||||
"\n",
|
||||
"# from .unilm.layoutlm.data.funsd import FunsdDataset, InputFeatures\n",
|
||||
"from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
|
||||
"\n",
|
||||
"batch_size = 16\n",
|
||||
"args = {'local_rank': -1,\n",
|
||||
" 'overwrite_cache': True,\n",
|
||||
" 'data_dir': '/home/sourab/temp/data/',\n",
|
||||
" 'model_name_or_path':'microsoft/layoutlm-base-uncased',\n",
|
||||
" 'max_seq_length': 512,\n",
|
||||
" 'model_type': 'layoutlm',\n",
|
||||
" }\n",
|
||||
"args = {\n",
|
||||
" \"local_rank\": -1,\n",
|
||||
" \"overwrite_cache\": True,\n",
|
||||
" \"data_dir\": \"/home/sourab/temp/data/\",\n",
|
||||
" \"model_name_or_path\": \"microsoft/layoutlm-base-uncased\",\n",
|
||||
" \"max_seq_length\": 512,\n",
|
||||
" \"model_type\": \"layoutlm\",\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# class to turn the keys of a dict into attributes (thanks Stackoverflow)\n",
|
||||
"class AttrDict(dict):\n",
|
||||
@@ -711,6 +702,7 @@
|
||||
" super(AttrDict, self).__init__(*args, **kwargs)\n",
|
||||
" self.__dict__ = self\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"args = AttrDict(args)\n",
|
||||
"\n",
|
||||
"tokenizer = LayoutLMTokenizer.from_pretrained(\"microsoft/layoutlm-base-uncased\")\n",
|
||||
@@ -718,15 +710,11 @@
|
||||
"# the LayoutLM authors already defined a specific FunsdDataset, so we are going to use this here\n",
|
||||
"train_dataset = FunsdDataset(args, tokenizer, labels, pad_token_label_id, mode=\"train\")\n",
|
||||
"train_sampler = RandomSampler(train_dataset)\n",
|
||||
"train_dataloader = DataLoader(train_dataset,\n",
|
||||
" sampler=train_sampler,\n",
|
||||
" batch_size=batch_size)\n",
|
||||
"train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=batch_size)\n",
|
||||
"\n",
|
||||
"eval_dataset = FunsdDataset(args, tokenizer, labels, pad_token_label_id, mode=\"test\")\n",
|
||||
"eval_sampler = SequentialSampler(eval_dataset)\n",
|
||||
"eval_dataloader = DataLoader(eval_dataset,\n",
|
||||
" sampler=eval_sampler,\n",
|
||||
" batch_size=batch_size)"
|
||||
"eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=batch_size)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -828,14 +816,10 @@
|
||||
],
|
||||
"source": [
|
||||
"from peft import get_peft_config, PeftModel, get_peft_model, LoraConfig, TaskType\n",
|
||||
"\n",
|
||||
"peft_config = LoraConfig(\n",
|
||||
" task_type=TaskType.TOKEN_CLS,\n",
|
||||
" inference_mode=False,\n",
|
||||
" r=16,\n",
|
||||
" lora_alpha=16,\n",
|
||||
" lora_dropout=0.1,\n",
|
||||
" bias=\"all\"\n",
|
||||
" )\n",
|
||||
" task_type=TaskType.TOKEN_CLS, inference_mode=False, r=16, lora_alpha=16, lora_dropout=0.1, bias=\"all\"\n",
|
||||
")\n",
|
||||
"peft_config"
|
||||
]
|
||||
},
|
||||
@@ -883,7 +867,7 @@
|
||||
"source": [
|
||||
"print(model.model.layoutlm.encoder.layer[0].attention.self.query.weight)\n",
|
||||
"print(model.model.layoutlm.encoder.layer[0].attention.self.query.lora_A.weight)\n",
|
||||
"print(model.model.classifier.weight)\n"
|
||||
"print(model.model.classifier.weight)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -909,50 +893,52 @@
|
||||
"source": [
|
||||
"from transformers import AdamW, get_linear_schedule_with_warmup\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"\n",
|
||||
"num_train_epochs = 100\n",
|
||||
"\n",
|
||||
"optimizer = torch.optim.AdamW(model.parameters(), lr=3e-3)\n",
|
||||
"lr_scheduler = get_linear_schedule_with_warmup(\n",
|
||||
" optimizer=optimizer,\n",
|
||||
" num_warmup_steps=0.06*(len(train_dataloader) * num_train_epochs),\n",
|
||||
" num_warmup_steps=0.06 * (len(train_dataloader) * num_train_epochs),\n",
|
||||
" num_training_steps=(len(train_dataloader) * num_train_epochs),\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"global_step = 0\n",
|
||||
"\n",
|
||||
"t_total = len(train_dataloader) * num_train_epochs # total number of training steps \n",
|
||||
"t_total = len(train_dataloader) * num_train_epochs # total number of training steps\n",
|
||||
"\n",
|
||||
"#put the model in training mode\n",
|
||||
"# put the model in training mode\n",
|
||||
"model.train()\n",
|
||||
"for epoch in range(num_train_epochs):\n",
|
||||
" for batch in tqdm(train_dataloader, desc=\"Training\"):\n",
|
||||
" input_ids = batch[0].to(device)\n",
|
||||
" bbox = batch[4].to(device)\n",
|
||||
" attention_mask = batch[1].to(device)\n",
|
||||
" token_type_ids = batch[2].to(device)\n",
|
||||
" labels = batch[3].to(device)\n",
|
||||
" for batch in tqdm(train_dataloader, desc=\"Training\"):\n",
|
||||
" input_ids = batch[0].to(device)\n",
|
||||
" bbox = batch[4].to(device)\n",
|
||||
" attention_mask = batch[1].to(device)\n",
|
||||
" token_type_ids = batch[2].to(device)\n",
|
||||
" labels = batch[3].to(device)\n",
|
||||
"\n",
|
||||
" # forward pass\n",
|
||||
" outputs = model(input_ids=input_ids, bbox=bbox, attention_mask=attention_mask, token_type_ids=token_type_ids,\n",
|
||||
" labels=labels)\n",
|
||||
" loss = outputs.loss\n",
|
||||
" # forward pass\n",
|
||||
" outputs = model(\n",
|
||||
" input_ids=input_ids, bbox=bbox, attention_mask=attention_mask, token_type_ids=token_type_ids, labels=labels\n",
|
||||
" )\n",
|
||||
" loss = outputs.loss\n",
|
||||
"\n",
|
||||
" # print loss every 100 steps\n",
|
||||
" if global_step % 10 == 0:\n",
|
||||
" print(f\"Loss after {global_step} steps: {loss.item()}\")\n",
|
||||
" # print loss every 100 steps\n",
|
||||
" if global_step % 10 == 0:\n",
|
||||
" print(f\"Loss after {global_step} steps: {loss.item()}\")\n",
|
||||
"\n",
|
||||
" # backward pass to get the gradients \n",
|
||||
" loss.backward()\n",
|
||||
" # backward pass to get the gradients\n",
|
||||
" loss.backward()\n",
|
||||
"\n",
|
||||
" #print(\"Gradients on classification head:\")\n",
|
||||
" #print(model.classifier.weight.grad[6,:].sum())\n",
|
||||
" # print(\"Gradients on classification head:\")\n",
|
||||
" # print(model.classifier.weight.grad[6,:].sum())\n",
|
||||
"\n",
|
||||
" # update\n",
|
||||
" optimizer.step()\n",
|
||||
" lr_scheduler.step()\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
" global_step += 1"
|
||||
" # update\n",
|
||||
" optimizer.step()\n",
|
||||
" lr_scheduler.step()\n",
|
||||
" optimizer.zero_grad()\n",
|
||||
" global_step += 1"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1006,8 +992,9 @@
|
||||
" labels = batch[3].to(device)\n",
|
||||
"\n",
|
||||
" # forward pass\n",
|
||||
" outputs = model(input_ids=input_ids, bbox=bbox, attention_mask=attention_mask, token_type_ids=token_type_ids,\n",
|
||||
" labels=labels)\n",
|
||||
" outputs = model(\n",
|
||||
" input_ids=input_ids, bbox=bbox, attention_mask=attention_mask, token_type_ids=token_type_ids, labels=labels\n",
|
||||
" )\n",
|
||||
" # get the loss and logits\n",
|
||||
" tmp_eval_loss = outputs.loss\n",
|
||||
" logits = outputs.logits\n",
|
||||
@@ -1021,9 +1008,7 @@
|
||||
" out_label_ids = labels.detach().cpu().numpy()\n",
|
||||
" else:\n",
|
||||
" preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)\n",
|
||||
" out_label_ids = np.append(\n",
|
||||
" out_label_ids, labels.detach().cpu().numpy(), axis=0\n",
|
||||
" )\n",
|
||||
" out_label_ids = np.append(out_label_ids, labels.detach().cpu().numpy(), axis=0)\n",
|
||||
"\n",
|
||||
"# compute average evaluation loss\n",
|
||||
"eval_loss = eval_loss / nb_eval_steps\n",
|
||||
@@ -1070,7 +1055,7 @@
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model.save_pretrained(\"peft_layoutlm\")\n"
|
||||
"model.save_pretrained(\"peft_layoutlm\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user