diff --git a/examples/causal_language_modeling/pet_lora_clm_accelerate_big_model_inference.ipynb b/examples/causal_language_modeling/pet_lora_clm_accelerate_big_model_inference.ipynb new file mode 100644 index 0000000..eeae829 --- /dev/null +++ b/examples/causal_language_modeling/pet_lora_clm_accelerate_big_model_inference.ipynb @@ -0,0 +1,2200 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "71fbfca2", + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoModelForCausalLM\n", + "from pet import get_pet_config,get_pet_model, get_pet_model_state_dict, set_pet_model_state_dict, LoRAConfig, TaskType, pet_model_load_and_dispatch\n", + "import torch\n", + "from datasets import load_dataset\n", + "import os\n", + "from transformers import AutoTokenizer\n", + "from torch.utils.data import DataLoader\n", + "from transformers import default_data_collator,get_linear_schedule_with_warmup\n", + "from tqdm import tqdm\n", + "from datasets import load_dataset\n", + "\n", + "device = \"cuda\"\n", + "model_name_or_path = \"bigscience/bloomz-7b1\"\n", + "tokenizer_name_or_path = \"bigscience/bloomz-7b1\"\n", + "pet_config = LoRAConfig(task_type=TaskType.CAUSAL_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1)\n", + "\n", + "dataset_name = \"twitter_complaints\"\n", + "checkpoint_name = \"/home/sourab/\"+f\"{dataset_name}_{model_name_or_path}_{pet_config.pet_type}_{pet_config.task_type}_v1.pt\".replace(\"/\", \"_\")\n", + "text_column = \"Tweet text\"\n", + "label_column = \"text_label\"\n", + "max_length=64\n", + "lr = 1e-3\n", + "num_epochs = 50\n", + "batch_size=8\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "e1a3648b", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Found cached dataset raft (/home/sourab/.cache/huggingface/datasets/ought___raft/twitter_complaints/1.1.0/79c4de1312c1e3730043f7db07179c914f48403101f7124e2fe336f6f54d9f84)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "92d0876af16b4525a124c79cf2da14b2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/2 [00:00100:\n", + " break\n", + "test_preds" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "b33be5e6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "@sho_help trying to watch Showtime on my iPad, keep getting network issues. Hulu and Netflix are working fine. Help?\n", + "{'input_ids': tensor([[227985, 5484, 915, 2566, 125474, 168916, 10343, 427, 24985,\n", + " 33642, 14167, 664, 2670, 99607, 15, 11874, 15980, 14218,\n", + " 17327, 17, 162966, 530, 59557, 1306, 10789, 15977, 17,\n", + " 98135, 34, 77658, 915, 210]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1]])}\n", + "tensor([[227985, 5484, 915, 2566, 125474, 168916, 10343, 427, 24985,\n", + " 33642, 14167, 664, 2670, 99607, 15, 11874, 15980, 14218,\n", + " 17327, 17, 162966, 530, 59557, 1306, 10789, 15977, 17,\n", + " 98135, 34, 77658, 915, 210, 16449, 5952, 3, 3,\n", + " 3, 3, 3, 3, 3, 3]])\n", + "['Tweet text : @sho_help trying to watch Showtime on my iPad, keep getting network issues. Hulu and Netflix are working fine. Help? Label : complaint']\n" + ] + } + ], + "source": [ + "model.eval()\n", + "i = 80\n", + "inputs = tokenizer(f'{text_column} : {dataset[\"test\"][i][\"Tweet text\"]} Label : ', return_tensors=\"pt\")\n", + "print(dataset[\"test\"][i][\"Tweet text\"])\n", + "print(inputs)\n", + "\n", + "with torch.no_grad():\n", + " outputs = model.generate(input_ids=inputs[\"input_ids\"], max_new_tokens=10)\n", + " print(outputs)\n", + " print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1c4ad9c", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/causal_language_modeling/pet_prefix_tuning_clm.ipynb b/examples/causal_language_modeling/pet_prefix_tuning_clm.ipynb new file mode 100644 index 0000000..33bf9d0 --- /dev/null +++ b/examples/causal_language_modeling/pet_prefix_tuning_clm.ipynb @@ -0,0 +1,2551 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "71fbfca2", + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoModelForCausalLM\n", + "from pet import get_pet_config,get_pet_model, get_pet_model_state_dict, set_pet_model_state_dict, PrefixTuningConfig, TaskType, pet_model_load_and_dispatch, bloom_model_postprocess_past_key_value, PETType\n", + "import torch\n", + "from datasets import load_dataset\n", + "import os\n", + "from transformers import AutoTokenizer\n", + "from torch.utils.data import DataLoader\n", + "from transformers import default_data_collator,get_linear_schedule_with_warmup\n", + "from tqdm import tqdm\n", + "from datasets import load_dataset\n", + "\n", + "device = \"cuda\"\n", + "model_name_or_path = \"bigscience/bloomz-560m\"\n", + "tokenizer_name_or_path = \"bigscience/bloomz-560m\"\n", + "pet_config = PrefixTuningConfig(task_type=TaskType.CAUSAL_LM, \n", + " num_virtual_tokens=30, \n", + " postprocess_past_key_value_function=bloom_model_postprocess_past_key_value)\n", + "\n", + "dataset_name = \"twitter_complaints\"\n", + "checkpoint_name = f\"{dataset_name}_{model_name_or_path}_{pet_config.pet_type}_{pet_config.task_type}_v1.pt\".replace(\"/\", \"_\")\n", + "text_column = \"Tweet text\"\n", + "label_column = \"text_label\"\n", + "max_length=64\n", + "lr = 3e-2\n", + "num_epochs = 50\n", + "batch_size=8\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "e1a3648b", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Found cached dataset raft (/home/sourab/.cache/huggingface/datasets/ought___raft/twitter_complaints/1.1.0/79c4de1312c1e3730043f7db07179c914f48403101f7124e2fe336f6f54d9f84)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ed9ea46fcc284122ab185639e102d430", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/2 [00:00, task_type=, inference_mode=False, num_virtual_tokens=30, token_dim=1024, num_transformer_submodules=1, num_attention_heads=16, num_layers=24, encoder_hidden_size=1024, prefix_projection=False, postprocess_past_key_value_function=)" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.pet_config" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "b2f91568", + "metadata": {}, + "outputs": [], + "source": [ + "# model\n", + "# optimizer and lr scheduler\n", + "optimizer = torch.optim.AdamW(model.parameters(), lr=lr)\n", + "lr_scheduler = get_linear_schedule_with_warmup(\n", + " optimizer=optimizer,\n", + " num_warmup_steps=0,\n", + " num_training_steps=(len(train_dataloader) * num_epochs),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "e4fb69fc", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:01<00:00, 5.93it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.43it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=0: train_ppl=tensor(4.1394e+09, device='cuda:0') train_epoch_loss=tensor(22.1438, device='cuda:0') eval_ppl=tensor(1402.4835, device='cuda:0') eval_epoch_loss=tensor(7.2460, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.39it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.49it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=1: train_ppl=tensor(246.8958, device='cuda:0') train_epoch_loss=tensor(5.5090, device='cuda:0') eval_ppl=tensor(62.6347, device='cuda:0') eval_epoch_loss=tensor(4.1373, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.40it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=2: train_ppl=tensor(40.7671, device='cuda:0') train_epoch_loss=tensor(3.7079, device='cuda:0') eval_ppl=tensor(20.4430, device='cuda:0') eval_epoch_loss=tensor(3.0176, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.37it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.52it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=3: train_ppl=tensor(13.3799, device='cuda:0') train_epoch_loss=tensor(2.5938, device='cuda:0') eval_ppl=tensor(7.8204, device='cuda:0') eval_epoch_loss=tensor(2.0567, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.44it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.55it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=4: train_ppl=tensor(5.6719, device='cuda:0') train_epoch_loss=tensor(1.7355, device='cuda:0') eval_ppl=tensor(3.2507, device='cuda:0') eval_epoch_loss=tensor(1.1789, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.44it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=5: train_ppl=tensor(2.4837, device='cuda:0') train_epoch_loss=tensor(0.9098, device='cuda:0') eval_ppl=tensor(1.5463, device='cuda:0') eval_epoch_loss=tensor(0.4359, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.53it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=6: train_ppl=tensor(1.4864, device='cuda:0') train_epoch_loss=tensor(0.3964, device='cuda:0') eval_ppl=tensor(1.8123, device='cuda:0') eval_epoch_loss=tensor(0.5946, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.45it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.49it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=7: train_ppl=tensor(1.4421, device='cuda:0') train_epoch_loss=tensor(0.3661, device='cuda:0') eval_ppl=tensor(1.6831, device='cuda:0') eval_epoch_loss=tensor(0.5206, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.38it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=8: train_ppl=tensor(1.5938, device='cuda:0') train_epoch_loss=tensor(0.4661, device='cuda:0') eval_ppl=tensor(1.4380, device='cuda:0') eval_epoch_loss=tensor(0.3632, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.44it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=9: train_ppl=tensor(1.2076, device='cuda:0') train_epoch_loss=tensor(0.1886, device='cuda:0') eval_ppl=tensor(1.2875, device='cuda:0') eval_epoch_loss=tensor(0.2527, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.51it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=10: train_ppl=tensor(1.2741, device='cuda:0') train_epoch_loss=tensor(0.2422, device='cuda:0') eval_ppl=tensor(1.2635, device='cuda:0') eval_epoch_loss=tensor(0.2339, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=11: train_ppl=tensor(1.2219, device='cuda:0') train_epoch_loss=tensor(0.2004, device='cuda:0') eval_ppl=tensor(1.1836, device='cuda:0') eval_epoch_loss=tensor(0.1686, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.52it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=12: train_ppl=tensor(1.1773, device='cuda:0') train_epoch_loss=tensor(0.1632, device='cuda:0') eval_ppl=tensor(1.1829, device='cuda:0') eval_epoch_loss=tensor(0.1680, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.41it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.53it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=13: train_ppl=tensor(1.1660, device='cuda:0') train_epoch_loss=tensor(0.1536, device='cuda:0') eval_ppl=tensor(1.1448, device='cuda:0') eval_epoch_loss=tensor(0.1353, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.51it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=14: train_ppl=tensor(1.1428, device='cuda:0') train_epoch_loss=tensor(0.1334, device='cuda:0') eval_ppl=tensor(1.1405, device='cuda:0') eval_epoch_loss=tensor(0.1315, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=15: train_ppl=tensor(1.1204, device='cuda:0') train_epoch_loss=tensor(0.1137, device='cuda:0') eval_ppl=tensor(1.1171, device='cuda:0') eval_epoch_loss=tensor(0.1108, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.50it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=16: train_ppl=tensor(1.1038, device='cuda:0') train_epoch_loss=tensor(0.0988, device='cuda:0') eval_ppl=tensor(1.0856, device='cuda:0') eval_epoch_loss=tensor(0.0821, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.53it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=17: train_ppl=tensor(1.0810, device='cuda:0') train_epoch_loss=tensor(0.0779, device='cuda:0') eval_ppl=tensor(1.0642, device='cuda:0') eval_epoch_loss=tensor(0.0623, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.41it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=18: train_ppl=tensor(1.0586, device='cuda:0') train_epoch_loss=tensor(0.0569, device='cuda:0') eval_ppl=tensor(1.0542, device='cuda:0') eval_epoch_loss=tensor(0.0528, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=19: train_ppl=tensor(1.0449, device='cuda:0') train_epoch_loss=tensor(0.0439, device='cuda:0') eval_ppl=tensor(1.0455, device='cuda:0') eval_epoch_loss=tensor(0.0445, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=20: train_ppl=tensor(1.0310, device='cuda:0') train_epoch_loss=tensor(0.0306, device='cuda:0') eval_ppl=tensor(1.0159, device='cuda:0') eval_epoch_loss=tensor(0.0158, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.42it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=21: train_ppl=tensor(1.0200, device='cuda:0') train_epoch_loss=tensor(0.0198, device='cuda:0') eval_ppl=tensor(1.0333, device='cuda:0') eval_epoch_loss=tensor(0.0327, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.46it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=22: train_ppl=tensor(1.0209, device='cuda:0') train_epoch_loss=tensor(0.0207, device='cuda:0') eval_ppl=tensor(1.0187, device='cuda:0') eval_epoch_loss=tensor(0.0185, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.40it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=23: train_ppl=tensor(1.0098, device='cuda:0') train_epoch_loss=tensor(0.0097, device='cuda:0') eval_ppl=tensor(1.0097, device='cuda:0') eval_epoch_loss=tensor(0.0096, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.45it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.50it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=24: train_ppl=tensor(1.0120, device='cuda:0') train_epoch_loss=tensor(0.0120, device='cuda:0') eval_ppl=tensor(1.0162, device='cuda:0') eval_epoch_loss=tensor(0.0161, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.49it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=25: train_ppl=tensor(1.0120, device='cuda:0') train_epoch_loss=tensor(0.0119, device='cuda:0') eval_ppl=tensor(1.0090, device='cuda:0') eval_epoch_loss=tensor(0.0090, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.40it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=26: train_ppl=tensor(1.0125, device='cuda:0') train_epoch_loss=tensor(0.0124, device='cuda:0') eval_ppl=tensor(1.0079, device='cuda:0') eval_epoch_loss=tensor(0.0079, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=27: train_ppl=tensor(1.0072, device='cuda:0') train_epoch_loss=tensor(0.0072, device='cuda:0') eval_ppl=tensor(1.0043, device='cuda:0') eval_epoch_loss=tensor(0.0043, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.44it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=28: train_ppl=tensor(1.0044, device='cuda:0') train_epoch_loss=tensor(0.0044, device='cuda:0') eval_ppl=tensor(1.0049, device='cuda:0') eval_epoch_loss=tensor(0.0048, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.51it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=29: train_ppl=tensor(1.0047, device='cuda:0') train_epoch_loss=tensor(0.0047, device='cuda:0') eval_ppl=tensor(1.0038, device='cuda:0') eval_epoch_loss=tensor(0.0038, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.41it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.52it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=30: train_ppl=tensor(1.0037, device='cuda:0') train_epoch_loss=tensor(0.0037, device='cuda:0') eval_ppl=tensor(1.0035, device='cuda:0') eval_epoch_loss=tensor(0.0035, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=31: train_ppl=tensor(1.0033, device='cuda:0') train_epoch_loss=tensor(0.0033, device='cuda:0') eval_ppl=tensor(1.0031, device='cuda:0') eval_epoch_loss=tensor(0.0031, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.48it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=32: train_ppl=tensor(1.0029, device='cuda:0') train_epoch_loss=tensor(0.0029, device='cuda:0') eval_ppl=tensor(1.0029, device='cuda:0') eval_epoch_loss=tensor(0.0029, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.45it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=33: train_ppl=tensor(1.0027, device='cuda:0') train_epoch_loss=tensor(0.0027, device='cuda:0') eval_ppl=tensor(1.0028, device='cuda:0') eval_epoch_loss=tensor(0.0028, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.50it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=34: train_ppl=tensor(1.0026, device='cuda:0') train_epoch_loss=tensor(0.0026, device='cuda:0') eval_ppl=tensor(1.0027, device='cuda:0') eval_epoch_loss=tensor(0.0027, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.49it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=35: train_ppl=tensor(1.0026, device='cuda:0') train_epoch_loss=tensor(0.0026, device='cuda:0') eval_ppl=tensor(1.0026, device='cuda:0') eval_epoch_loss=tensor(0.0026, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.30it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.03it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=36: train_ppl=tensor(1.0025, device='cuda:0') train_epoch_loss=tensor(0.0025, device='cuda:0') eval_ppl=tensor(1.0025, device='cuda:0') eval_epoch_loss=tensor(0.0025, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=37: train_ppl=tensor(1.0024, device='cuda:0') train_epoch_loss=tensor(0.0023, device='cuda:0') eval_ppl=tensor(1.0024, device='cuda:0') eval_epoch_loss=tensor(0.0024, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.40it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=38: train_ppl=tensor(1.0023, device='cuda:0') train_epoch_loss=tensor(0.0023, device='cuda:0') eval_ppl=tensor(1.0023, device='cuda:0') eval_epoch_loss=tensor(0.0023, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.47it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=39: train_ppl=tensor(1.0023, device='cuda:0') train_epoch_loss=tensor(0.0023, device='cuda:0') eval_ppl=tensor(1.0023, device='cuda:0') eval_epoch_loss=tensor(0.0023, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.51it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=40: train_ppl=tensor(1.0022, device='cuda:0') train_epoch_loss=tensor(0.0022, device='cuda:0') eval_ppl=tensor(1.0023, device='cuda:0') eval_epoch_loss=tensor(0.0022, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.46it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=41: train_ppl=tensor(1.0022, device='cuda:0') train_epoch_loss=tensor(0.0022, device='cuda:0') eval_ppl=tensor(1.0022, device='cuda:0') eval_epoch_loss=tensor(0.0022, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.43it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=42: train_ppl=tensor(1.0021, device='cuda:0') train_epoch_loss=tensor(0.0021, device='cuda:0') eval_ppl=tensor(1.0022, device='cuda:0') eval_epoch_loss=tensor(0.0022, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.49it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=43: train_ppl=tensor(1.0022, device='cuda:0') train_epoch_loss=tensor(0.0022, device='cuda:0') eval_ppl=tensor(1.0021, device='cuda:0') eval_epoch_loss=tensor(0.0021, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.43it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.46it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=44: train_ppl=tensor(1.0021, device='cuda:0') train_epoch_loss=tensor(0.0021, device='cuda:0') eval_ppl=tensor(1.0021, device='cuda:0') eval_epoch_loss=tensor(0.0021, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.42it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.40it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=45: train_ppl=tensor(1.0020, device='cuda:0') train_epoch_loss=tensor(0.0020, device='cuda:0') eval_ppl=tensor(1.0021, device='cuda:0') eval_epoch_loss=tensor(0.0021, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.41it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.43it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=46: train_ppl=tensor(1.0021, device='cuda:0') train_epoch_loss=tensor(0.0021, device='cuda:0') eval_ppl=tensor(1.0021, device='cuda:0') eval_epoch_loss=tensor(0.0021, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.40it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.45it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=47: train_ppl=tensor(1.0021, device='cuda:0') train_epoch_loss=tensor(0.0021, device='cuda:0') eval_ppl=tensor(1.0021, device='cuda:0') eval_epoch_loss=tensor(0.0021, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.41it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.43it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=48: train_ppl=tensor(1.0020, device='cuda:0') train_epoch_loss=tensor(0.0020, device='cuda:0') eval_ppl=tensor(1.0021, device='cuda:0') eval_epoch_loss=tensor(0.0021, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 11.41it/s]\n", + "100%|█████████████████████████████████████████████████████████████████| 7/7 [00:00<00:00, 22.42it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch=49: train_ppl=tensor(1.0020, device='cuda:0') train_epoch_loss=tensor(0.0020, device='cuda:0') eval_ppl=tensor(1.0021, device='cuda:0') eval_epoch_loss=tensor(0.0021, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "# training and evaluation\n", + "model = model.to(device)\n", + "\n", + "for epoch in range(num_epochs):\n", + " model.train()\n", + " total_loss = 0\n", + " for step, batch in enumerate(tqdm(train_dataloader)):\n", + " batch = {k: v.to(device) for k, v in batch.items()}\n", + "# print(batch)\n", + "# print(batch[\"input_ids\"].shape)\n", + " outputs = model(**batch)\n", + " loss = outputs.loss\n", + " total_loss += loss.detach().float()\n", + " loss.backward()\n", + " optimizer.step()\n", + " lr_scheduler.step()\n", + " optimizer.zero_grad()\n", + "\n", + " model.eval()\n", + " eval_loss = 0\n", + " eval_preds = []\n", + " for step, batch in enumerate(tqdm(eval_dataloader)):\n", + " batch = {k: v.to(device) for k, v in batch.items()}\n", + " with torch.no_grad():\n", + " outputs = model(**batch)\n", + " loss = outputs.loss\n", + " eval_loss += loss.detach().float()\n", + " eval_preds.extend(tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True))\n", + "\n", + " eval_epoch_loss = eval_loss/len(train_dataloader)\n", + " eval_ppl = torch.exp(eval_epoch_loss)\n", + " train_epoch_loss = total_loss/len(eval_dataloader)\n", + " train_ppl = torch.exp(train_epoch_loss)\n", + " print(f\"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "53752a7b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "@VW @QuirkCars some unauthorized work done on my engine now throwing a check engine light about a week later. Very upset.\n", + "{'input_ids': tensor([[227985, 5484, 915, 2566, 57, 58, 2566, 5232, 132511,\n", + " 38, 4599, 3331, 1035, 192352, 2909, 11541, 664, 2670,\n", + " 22218, 5840, 108218, 267, 7010, 22218, 12490, 3638, 267,\n", + " 14319, 10494, 17, 93269, 123055, 17, 77658, 915, 210]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n", + "tensor([[227985, 5484, 915, 2566, 57, 58, 2566, 5232, 132511,\n", + " 38, 4599, 3331, 1035, 192352, 2909, 11541, 664, 2670,\n", + " 22218, 5840, 108218, 267, 7010, 22218, 12490, 3638, 267,\n", + " 14319, 10494, 17, 93269, 123055, 17, 77658, 915, 210,\n", + " 16449, 5952, 3, 3, 3, 3, 3, 3, 3,\n", + " 3]], device='cuda:0')\n", + "['Tweet text : @VW @QuirkCars some unauthorized work done on my engine now throwing a check engine light about a week later. Very upset. Label : complaint']\n" + ] + } + ], + "source": [ + "model.eval()\n", + "i = 12\n", + "inputs = tokenizer(f'{text_column} : {dataset[\"test\"][i][\"Tweet text\"]} Label : ', return_tensors=\"pt\")\n", + "print(dataset[\"test\"][i][\"Tweet text\"])\n", + "print(inputs)\n", + "\n", + "with torch.no_grad():\n", + " inputs = {k: v.to(device) for k, v in inputs.items()}\n", + " outputs = model.generate(input_ids=inputs[\"input_ids\"], attention_mask=inputs[\"attention_mask\"], max_new_tokens=10)\n", + " print(outputs)\n", + " print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8ba1f8c", + "metadata": {}, + "outputs": [], + "source": [ + "model.eval()\n", + "eval_loss = 0\n", + "eval_preds = []\n", + "for step, batch in enumerate(tqdm(test_dataloader)):\n", + " batch = {k: v.to(device) for k, v in batch.items() if k!=\"labels\"}\n", + " with torch.no_grad():\n", + " outputs = model.generate(**batch, max_new_tokens=10)\n", + " eval_preds.extend(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "252f733d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'prompt_embeddings': tensor([[ 0.7356, -1.0849, -0.4560, ..., 1.0242, 0.3908, -0.8000],\n", + " [-1.5587, -0.3595, 0.1289, ..., 0.5427, 1.0976, 1.7641],\n", + " [-0.2113, -1.4675, -0.5976, ..., 0.1691, -0.5843, -0.2658],\n", + " ...,\n", + " [ 2.1275, 0.7253, 0.0323, ..., -1.2285, -0.5614, 0.0370],\n", + " [-0.2258, -1.5149, 0.0685, ..., -1.4476, -0.1348, -0.6910],\n", + " [ 0.9089, 0.3947, -1.5271, ..., 1.9079, 0.6473, 0.7306]])}\n" + ] + } + ], + "source": [ + "# saving model\n", + "state_dict = get_pet_model_state_dict(model)\n", + "torch.save(state_dict, checkpoint_name)\n", + "print(state_dict)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "4928c7f1", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "5,7M\ttwitter_complaints_bigscience_bloomz-560m_PREFIX_TUNING_CAUSAL_LM_v1.pt\r\n" + ] + } + ], + "source": [ + "!du -h $checkpoint_name" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "4d9476e1", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "PrefixTuningConfig(pet_type=, task_type=, inference_mode=True, num_virtual_tokens=30, token_dim=None, num_transformer_submodules=1, num_attention_heads=None, num_layers=None, encoder_hidden_size=None, prefix_projection=False, postprocess_past_key_value_function=)\n", + "trainable params: 1474560 || all params: 560689152 || trainable%: 0.26299064191632515\n" + ] + } + ], + "source": [ + "max_memory={0: \"1GIB\", 1: \"1GIB\", 2: \"2GIB\", 3: \"2GIB\", \"cpu\":\"30GB\"}\n", + "\n", + "pet_config.inference_mode = True\n", + "print(pet_config)\n", + "model = AutoModelForCausalLM.from_pretrained(model_name_or_path, device_map=\"auto\", max_memory=max_memory)\n", + "model = pet_model_load_and_dispatch(model, torch.load(checkpoint_name), pet_config, max_memory)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "5c7b3d71", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "PETModelForCausalLM(\n", + " (base_model): BloomForCausalLM(\n", + " (transformer): BloomModel(\n", + " (word_embeddings): Embedding(250880, 1024)\n", + " (word_embeddings_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (h): ModuleList(\n", + " (0): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (1): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (2): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (3): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (4): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (5): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (6): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (7): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (8): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (9): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (10): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (11): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (12): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (13): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (14): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (15): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (16): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (17): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (18): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (19): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (20): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (21): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (22): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " (23): BloomBlock(\n", + " (input_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (self_attention): BloomAttention(\n", + " (query_key_value): Linear(in_features=1024, out_features=3072, bias=True)\n", + " (dense): Linear(in_features=1024, out_features=1024, bias=True)\n", + " (attention_dropout): Dropout(p=0.0, inplace=False)\n", + " )\n", + " (post_attention_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " (mlp): BloomMLP(\n", + " (dense_h_to_4h): Linear(in_features=1024, out_features=4096, bias=True)\n", + " (gelu_impl): BloomGelu()\n", + " (dense_4h_to_h): Linear(in_features=4096, out_features=1024, bias=True)\n", + " )\n", + " )\n", + " )\n", + " (ln_f): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n", + " )\n", + " (lm_head): Linear(in_features=1024, out_features=250880, bias=False)\n", + " )\n", + " (word_embeddings): Embedding(250880, 1024)\n", + " (prompt_encoder): PrefixEncoder(\n", + " (embedding): Embedding(30, 49152)\n", + " )\n", + ")" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "b94465ee", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'base_model.transformer.word_embeddings': 3,\n", + " 'word_embeddings': 3,\n", + " 'base_model.transformer.word_embeddings_layernorm': 3,\n", + " 'base_model.transformer.h.0': 3,\n", + " 'base_model.transformer.h.1': 3,\n", + " 'base_model.transformer.h.2': 3,\n", + " 'base_model.transformer.h.3': 3,\n", + " 'base_model.transformer.h.4': 3,\n", + " 'base_model.transformer.h.5': 3,\n", + " 'base_model.transformer.h.6': 3,\n", + " 'base_model.transformer.h.7': 3,\n", + " 'base_model.transformer.h.8': 3,\n", + " 'base_model.transformer.h.9': 3,\n", + " 'base_model.transformer.h.10': 3,\n", + " 'base_model.transformer.h.11': 3,\n", + " 'base_model.transformer.h.12': 3,\n", + " 'base_model.transformer.h.13': 3,\n", + " 'base_model.transformer.h.14': 3,\n", + " 'base_model.transformer.h.15': 3,\n", + " 'base_model.transformer.h.16': 3,\n", + " 'base_model.transformer.h.17': 3,\n", + " 'base_model.transformer.h.18': 3,\n", + " 'base_model.transformer.h.19': 3,\n", + " 'base_model.transformer.h.20': 3,\n", + " 'base_model.transformer.h.21': 3,\n", + " 'base_model.transformer.h.22': 'cpu',\n", + " 'base_model.transformer.h.23': 'cpu',\n", + " 'base_model.transformer.ln_f': 'cpu',\n", + " 'base_model.lm_head': 'cpu',\n", + " 'prompt_encoder': 'cpu'}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.hf_device_map" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "ebe174a6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "@VW @QuirkCars some unauthorized work done on my engine now throwing a check engine light about a week later. Very upset.\n", + "{'input_ids': tensor([[227985, 5484, 915, 2566, 57, 58, 2566, 5232, 132511,\n", + " 38, 4599, 3331, 1035, 192352, 2909, 11541, 664, 2670,\n", + " 22218, 5840, 108218, 267, 7010, 22218, 12490, 3638, 267,\n", + " 14319, 10494, 17, 93269, 123055, 17, 77658, 915, 210]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n", + "tensor([[227985, 5484, 915, 2566, 57, 58, 2566, 5232, 132511,\n", + " 38, 4599, 3331, 1035, 192352, 2909, 11541, 664, 2670,\n", + " 22218, 5840, 108218, 267, 7010, 22218, 12490, 3638, 267,\n", + " 14319, 10494, 17, 93269, 123055, 17, 77658, 915, 210,\n", + " 16449, 5952, 3, 3, 3, 3, 3, 3, 3,\n", + " 3]], device='cuda:0')\n", + "['Tweet text : @VW @QuirkCars some unauthorized work done on my engine now throwing a check engine light about a week later. Very upset. Label : complaint']\n" + ] + } + ], + "source": [ + "model.eval()\n", + "i = 12\n", + "inputs = tokenizer(f'{text_column} : {dataset[\"test\"][i][\"Tweet text\"]} Label : ', return_tensors=\"pt\")\n", + "print(dataset[\"test\"][i][\"Tweet text\"])\n", + "print(inputs)\n", + "\n", + "with torch.no_grad():\n", + " inputs = {k: v.to(device) for k, v in inputs.items()}\n", + " outputs = model.generate(input_ids=inputs[\"input_ids\"], attention_mask=inputs[\"attention_mask\"], max_new_tokens=10)\n", + " print(outputs)\n", + " print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24041ee1", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 (clang-1300.0.29.30)]" + }, + "vscode": { + "interpreter": { + "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49" + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/pet/__init__.py b/src/pet/__init__.py index 2a48832..d7f7175 100644 --- a/src/pet/__init__.py +++ b/src/pet/__init__.py @@ -31,7 +31,7 @@ from .utils import ( TaskType, bloom_model_postprocess_past_key_value, get_pet_model_state_dict, + pet_model_load_and_dispatch, set_pet_model_state_dict, shift_tokens_right, - pet_model_load_and_dispatch, ) diff --git a/src/pet/utils/__init__.py b/src/pet/utils/__init__.py index af3188e..3d1cc4e 100644 --- a/src/pet/utils/__init__.py +++ b/src/pet/utils/__init__.py @@ -4,4 +4,4 @@ from .config import PETConfig, PETType, PromptLearningConfig, TaskType from .other import _set_trainable, bloom_model_postprocess_past_key_value, shift_tokens_right -from .save_and_load import get_pet_model_state_dict, set_pet_model_state_dict, pet_model_load_and_dispatch +from .save_and_load import get_pet_model_state_dict, pet_model_load_and_dispatch, set_pet_model_state_dict diff --git a/src/pet/utils/save_and_load.py b/src/pet/utils/save_and_load.py index 6c5580b..4102b4a 100644 --- a/src/pet/utils/save_and_load.py +++ b/src/pet/utils/save_and_load.py @@ -88,5 +88,6 @@ def pet_model_load_and_dispatch(model, pet_model_state_dict, pet_config, max_mem if model.pet_config.pet_type == PETType.LORA: add_hook_to_module(model.base_model.model, hook) else: + remove_hook_from_submodules(model.prompt_encoder) add_hook_to_module(model.base_model, hook) return model