From afb171eefb3b252471ba117ec93585e7bdd00e09 Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Wed, 8 Feb 2023 18:07:15 +0530 Subject: [PATCH] fixes and updating examples --- ...a_clm_accelerate_big_model_inference.ipynb | 2 +- ...ccelerate_ds_zero3_cpu_offload_config.yaml | 22 + .../peft_lora_seq2seq.ipynb | 142 ++++-- ...q2seq_accelerate_big_model_inference.ipynb | 463 ++++++++++++++++++ .../peft_prefix_tuning_seq2seq.ipynb | 188 +++---- src/peft/peft_model.py | 18 +- 6 files changed, 711 insertions(+), 124 deletions(-) create mode 100644 examples/conditional_generation/accelerate_ds_zero3_cpu_offload_config.yaml create mode 100644 examples/conditional_generation/peft_lora_seq2seq_accelerate_big_model_inference.ipynb diff --git a/examples/causal_language_modeling/peft_lora_clm_accelerate_big_model_inference.ipynb b/examples/causal_language_modeling/peft_lora_clm_accelerate_big_model_inference.ipynb index d500099..b4c69ee 100644 --- a/examples/causal_language_modeling/peft_lora_clm_accelerate_big_model_inference.ipynb +++ b/examples/causal_language_modeling/peft_lora_clm_accelerate_big_model_inference.ipynb @@ -510,7 +510,7 @@ "\n", "config = PeftConfig.from_pretrained(peft_model_id)\n", "model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path, device_map=\"auto\", max_memory=max_memory)\n", - "model = PeftModel.from_pretrained(model, peft_model_id, max_memory=max_memory)" + "model = PeftModel.from_pretrained(model, peft_model_id, device_map=\"auto\", max_memory=max_memory)" ] }, { diff --git a/examples/conditional_generation/accelerate_ds_zero3_cpu_offload_config.yaml b/examples/conditional_generation/accelerate_ds_zero3_cpu_offload_config.yaml new file mode 100644 index 0000000..a4a0bcf --- /dev/null +++ b/examples/conditional_generation/accelerate_ds_zero3_cpu_offload_config.yaml @@ -0,0 +1,22 @@ +compute_environment: LOCAL_MACHINE +deepspeed_config: + gradient_accumulation_steps: 1 + gradient_clipping: 1.0 + offload_optimizer_device: none + offload_param_device: none + zero3_init_flag: true + zero3_save_16bit_model: true + zero_stage: 3 +distributed_type: DEEPSPEED +downcast_bf16: 'no' +dynamo_backend: 'NO' +fsdp_config: {} +machine_rank: 0 +main_training_function: main +megatron_lm_config: {} +mixed_precision: 'no' +num_machines: 1 +num_processes: 1 +rdzv_backend: static +same_network: true +use_cpu: false \ No newline at end of file diff --git a/examples/conditional_generation/peft_lora_seq2seq.ipynb b/examples/conditional_generation/peft_lora_seq2seq.ipynb index 55266c7..c308738 100644 --- a/examples/conditional_generation/peft_lora_seq2seq.ipynb +++ b/examples/conditional_generation/peft_lora_seq2seq.ipynb @@ -2,10 +2,26 @@ "cells": [ { "cell_type": "code", - "execution_count": 17, + "execution_count": 1, "id": "5f93b7d1", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===================================BUG REPORT===================================\n", + "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n", + "For effortless bug reporting copy-paste your error into this form: https://docs.google.com/forms/d/e/1FAIpQLScPB8emS3Thkp66nvqwmjTEgxp8Y9ufuWTzFyr9kJ5AoI47dQ/viewform?usp=sf_link\n", + "================================================================================\n", + "CUDA SETUP: CUDA runtime path found: /home/sourab/miniconda3/envs/ml/lib/libcudart.so\n", + "CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n", + "CUDA SETUP: Detected CUDA version 117\n", + "CUDA SETUP: Loading binary /home/sourab/miniconda3/envs/ml/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n" + ] + } + ], "source": [ "from transformers import AutoModelForSeq2SeqLM\n", "from peft import get_peft_config,get_peft_model, get_peft_model_state_dict, LoraConfig, TaskType\n", @@ -60,15 +76,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "/home/sourab/miniconda3/envs/ml/lib/python3.10/site-packages/huggingface_hub/utils/_deprecation.py:97: FutureWarning: Deprecated argument(s) used in 'dataset_info': token. Will not be supported from version '0.12'.\n", - " warnings.warn(message, FutureWarning)\n", "Found cached dataset financial_phrasebank (/home/sourab/.cache/huggingface/datasets/financial_phrasebank/sentences_allagree/1.0.0/550bde12e6c30e2674da973a55f57edde5181d53f5a5a34c1531c53f93b7e141)\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6de075f8208349108291ac5ab7f5c980", + "model_id": "3403bf3d718042018b0531848cc30209", "version_major": 2, "version_minor": 0 }, @@ -82,7 +96,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4b0e67b6d93f43e4b0f6a2f8978e4b0c", + "model_id": "d3d5c45e3776469f9560b6eaa9346f8f", "version_major": 2, "version_minor": 0 }, @@ -96,7 +110,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a9551029c9884529bda7421a99170b51", + "model_id": "e9736f26e9aa450b8d65f95c0b9c81cc", "version_major": 2, "version_minor": 0 }, @@ -110,7 +124,7 @@ { "data": { "text/plain": [ - "{'sentence': 'The order was valued at USD12 .2 m.',\n", + "{'sentence': \"The 10,000-odd square metre plot that Stockmann has bought for the Nevsky Center shopping center is located on Nevsky Prospect , St Petersburg 's high street , next to the Vosstaniya Square underground station , in the immediate vicinity of Moscow Station .\",\n", " 'label': 1,\n", " 'text_label': 'neutral'}" ] @@ -147,7 +161,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4421971232434db1b6141e91fda2f6d7", + "model_id": "c460989d4ab24e3f97d81ef040b1d1b4", "version_major": 2, "version_minor": 0 }, @@ -161,7 +175,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "9b2ef793d93443949f4a5d5874d4bc05", + "model_id": "1acc389b08b94f8a87900b9fbdbccce4", "version_major": 2, "version_minor": 0 }, @@ -234,45 +248,52 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████| 255/255 [00:53<00:00, 4.80it/s]\n", - "100%|███████████████████████████████████████████████████████████████| 29/29 [00:02<00:00, 14.16it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████| 255/255 [02:21<00:00, 1.81it/s]\n", + "100%|██████████████████████████████████████████████████████████████████████████████████████████| 29/29 [00:07<00:00, 4.13it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=0: train_ppl=tensor(13.6966, device='cuda:0') train_epoch_loss=tensor(2.6171, device='cuda:0') eval_ppl=tensor(1.0046, device='cuda:0') eval_epoch_loss=tensor(0.0046, device='cuda:0')\n" + "epoch=0: train_ppl=tensor(14.6341, device='cuda:0') train_epoch_loss=tensor(2.6834, device='cuda:0') eval_ppl=tensor(1.0057, device='cuda:0') eval_epoch_loss=tensor(0.0057, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████| 255/255 [00:52<00:00, 4.88it/s]\n", - "100%|███████████████████████████████████████████████████████████████| 29/29 [00:02<00:00, 14.20it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████| 255/255 [02:00<00:00, 2.11it/s]\n", + "100%|██████████████████████████████████████████████████████████████████████████████████████████| 29/29 [00:05<00:00, 5.66it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=1: train_ppl=tensor(1.5893, device='cuda:0') train_epoch_loss=tensor(0.4633, device='cuda:0') eval_ppl=tensor(1.0020, device='cuda:0') eval_epoch_loss=tensor(0.0020, device='cuda:0')\n" + "epoch=1: train_ppl=tensor(1.7576, device='cuda:0') train_epoch_loss=tensor(0.5640, device='cuda:0') eval_ppl=tensor(1.0052, device='cuda:0') eval_epoch_loss=tensor(0.0052, device='cuda:0')\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "100%|█████████████████████████████████████████████████████████████| 255/255 [00:52<00:00, 4.87it/s]\n", - "100%|███████████████████████████████████████████████████████████████| 29/29 [00:02<00:00, 14.18it/s]\n" + "100%|████████████████████████████████████████████████████████████████████████████████████████| 255/255 [01:33<00:00, 2.74it/s]\n", + "100%|██████████████████████████████████████████████████████████████████████████████████████████| 29/29 [00:04<00:00, 6.23it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "epoch=2: train_ppl=tensor(1.3210, device='cuda:0') train_epoch_loss=tensor(0.2784, device='cuda:0') eval_ppl=tensor(1.0026, device='cuda:0') eval_epoch_loss=tensor(0.0026, device='cuda:0')\n" + "epoch=2: train_ppl=tensor(1.3830, device='cuda:0') train_epoch_loss=tensor(0.3243, device='cuda:0') eval_ppl=tensor(1.0035, device='cuda:0') eval_epoch_loss=tensor(0.0035, device='cuda:0')\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" ] } ], @@ -313,7 +334,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 7, "id": "6cafa67b", "metadata": {}, "outputs": [ @@ -321,9 +342,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "accuracy=98.23788546255507 % on the evaluation dataset\n", - "eval_preds[:10]=['neutral', 'neutral', 'positive', 'positive', 'neutral', 'neutral', 'neutral', 'neutral', 'neutral', 'neutral']\n", - "dataset['validation']['text_label'][:10]=['neutral', 'neutral', 'positive', 'positive', 'neutral', 'neutral', 'neutral', 'neutral', 'neutral', 'neutral']\n" + "accuracy=97.3568281938326 % on the evaluation dataset\n", + "eval_preds[:10]=['neutral', 'neutral', 'neutral', 'positive', 'neutral', 'positive', 'positive', 'neutral', 'neutral', 'neutral']\n", + "dataset['validation']['text_label'][:10]=['neutral', 'neutral', 'neutral', 'positive', 'neutral', 'positive', 'positive', 'neutral', 'neutral', 'neutral']\n" ] } ], @@ -343,20 +364,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "a8de6005", "metadata": {}, "outputs": [], "source": [ "# saving model\n", - "state_dict = get_peft_model_state_dict(model)\n", - "torch.save(state_dict, checkpoint_name)\n", - "print(state_dict)" + "peft_model_id = f\"{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}\"\n", + "model.save_pretrained(peft_model_id)" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 9, "id": "bd20cd4c", "metadata": {}, "outputs": [ @@ -364,18 +384,74 @@ "name": "stdout", "output_type": "stream", "text": [ - "19M\tfinancial_sentiment_analysis_lora_v1.pt\r\n" + "9,2M\tbigscience/mt0-large_LORA_SEQ_2_SEQ_LM/adapter_model.bin\r\n" ] } ], "source": [ - "!du -h $checkpoint_name" + "ckpt = f\"{peft_model_id}/adapter_model.bin\"\n", + "!du -h $ckpt" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "76c2fc29", + "metadata": {}, + "outputs": [], + "source": [ + "from peft import PeftModel, PeftConfig\n", + "peft_model_id = f\"{model_name_or_path}_{peft_config.peft_type}_{peft_config.task_type}\"\n", + "\n", + "config = PeftConfig.from_pretrained(peft_model_id)\n", + "model = AutoModelForSeq2SeqLM.from_pretrained(config.base_model_name_or_path)\n", + "model = PeftModel.from_pretrained(model, peft_model_id)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "37d712ce", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "- Demand for fireplace products was lower than expected , especially in Germany .\n", + "{'input_ids': tensor([[ 259, 264, 259, 82903, 332, 1090, 10040, 10371, 639, 259,\n", + " 19540, 2421, 259, 25505, 259, 261, 259, 21230, 281, 17052,\n", + " 259, 260, 1]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}\n", + "tensor([[ 0, 259, 32588, 1]])\n", + "['negative']\n" + ] + } + ], + "source": [ + "model.eval()\n", + "i = 13\n", + "inputs = tokenizer(dataset[\"validation\"][text_column][i], return_tensors=\"pt\")\n", + "print(dataset[\"validation\"][text_column][i])\n", + "print(inputs)\n", + "\n", + "with torch.no_grad():\n", + " outputs = model.generate(input_ids=inputs[\"input_ids\"], max_new_tokens=10)\n", + " print(outputs)\n", + " print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True))\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "76c2fc29", + "id": "66c65ea4", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "65e71f78", "metadata": {}, "outputs": [], "source": [] @@ -383,7 +459,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10.5 64-bit", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -397,7 +473,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 (clang-1300.0.29.30)]" + "version": "3.10.4" }, "vscode": { "interpreter": { diff --git a/examples/conditional_generation/peft_lora_seq2seq_accelerate_big_model_inference.ipynb b/examples/conditional_generation/peft_lora_seq2seq_accelerate_big_model_inference.ipynb new file mode 100644 index 0000000..ff7d89d --- /dev/null +++ b/examples/conditional_generation/peft_lora_seq2seq_accelerate_big_model_inference.ipynb @@ -0,0 +1,463 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "71fbfca2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===================================BUG REPORT===================================\n", + "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n", + "For effortless bug reporting copy-paste your error into this form: https://docs.google.com/forms/d/e/1FAIpQLScPB8emS3Thkp66nvqwmjTEgxp8Y9ufuWTzFyr9kJ5AoI47dQ/viewform?usp=sf_link\n", + "================================================================================\n", + "CUDA SETUP: CUDA runtime path found: /home/sourab/miniconda3/envs/ml/lib/libcudart.so\n", + "CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n", + "CUDA SETUP: Detected CUDA version 117\n", + "CUDA SETUP: Loading binary /home/sourab/miniconda3/envs/ml/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n" + ] + } + ], + "source": [ + "from transformers import AutoModelForSeq2SeqLM\n", + "from peft import PeftModel, PeftConfig\n", + "import torch\n", + "from datasets import load_dataset\n", + "import os\n", + "from transformers import AutoTokenizer\n", + "from torch.utils.data import DataLoader\n", + "from transformers import default_data_collator,get_linear_schedule_with_warmup\n", + "from tqdm import tqdm\n", + "from datasets import load_dataset\n", + "\n", + "dataset_name = \"twitter_complaints\"\n", + "text_column = \"Tweet text\"\n", + "label_column = \"text_label\"\n", + "batch_size=8\n", + "\n", + "peft_model_id = \"smangrul/twitter_complaints_bigscience_T0_3B_LORA_SEQ_2_SEQ_LM\"\n", + "config = PeftConfig.from_pretrained(peft_model_id)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "cc55820a", + "metadata": {}, + "outputs": [], + "source": [ + "peft_model_id = \"smangrul/twitter_complaints_bigscience_T0_3B_LORA_SEQ_2_SEQ_LM\"\n", + "max_memory={0: \"6GIB\", 1: \"0GIB\", 2: \"0GIB\", 3: \"0GIB\", 4: \"0GIB\", \"cpu\":\"30GB\"}\n", + "config = PeftConfig.from_pretrained(peft_model_id)\n", + "model = AutoModelForSeq2SeqLM.from_pretrained(config.base_model_name_or_path, device_map=\"auto\", max_memory=max_memory)\n", + "model = PeftModel.from_pretrained(model, peft_model_id, device_map=\"auto\", max_memory=max_memory)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "e1a3648b", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Found cached dataset raft (/home/sourab/.cache/huggingface/datasets/ought___raft/twitter_complaints/1.1.0/79c4de1312c1e3730043f7db07179c914f48403101f7124e2fe336f6f54d9f84)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0bf24d09ce9642ec804aa0f208f54e1d", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/2 [00:00100:\n", + " break\n", + "test_preds" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.4" + }, + "vscode": { + "interpreter": { + "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49" + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/conditional_generation/peft_prefix_tuning_seq2seq.ipynb b/examples/conditional_generation/peft_prefix_tuning_seq2seq.ipynb index 0f71eaf..b514d57 100644 --- a/examples/conditional_generation/peft_prefix_tuning_seq2seq.ipynb +++ b/examples/conditional_generation/peft_prefix_tuning_seq2seq.ipynb @@ -5,7 +5,23 @@ "execution_count": 1, "id": "5f93b7d1", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "===================================BUG REPORT===================================\n", + "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n", + "For effortless bug reporting copy-paste your error into this form: https://docs.google.com/forms/d/e/1FAIpQLScPB8emS3Thkp66nvqwmjTEgxp8Y9ufuWTzFyr9kJ5AoI47dQ/viewform?usp=sf_link\n", + "================================================================================\n", + "CUDA SETUP: CUDA runtime path found: /home/sourab/miniconda3/envs/ml/lib/libcudart.so\n", + "CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n", + "CUDA SETUP: Detected CUDA version 117\n", + "CUDA SETUP: Loading binary /home/sourab/miniconda3/envs/ml/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda117.so...\n" + ] + } + ], "source": [ "from transformers import AutoModelForSeq2SeqLM\n", "from peft import get_peft_config,get_peft_model, get_peft_model_state_dict, PrefixTuningConfig, TaskType\n", @@ -61,15 +77,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "/home/sourab/miniconda3/envs/ml/lib/python3.10/site-packages/huggingface_hub/utils/_deprecation.py:97: FutureWarning: Deprecated argument(s) used in 'dataset_info': token. Will not be supported from version '0.12'.\n", - " warnings.warn(message, FutureWarning)\n", "Found cached dataset financial_phrasebank (/home/sourab/.cache/huggingface/datasets/financial_phrasebank/sentences_allagree/1.0.0/550bde12e6c30e2674da973a55f57edde5181d53f5a5a34c1531c53f93b7e141)\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e3f8b8faca0a4112b2c3499faee9544b", + "model_id": "ec4be98991b84181bfa75f8846422b8b", "version_major": 2, "version_minor": 0 }, @@ -83,7 +97,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "935c8aebde284a5784348588e0bb013a", + "model_id": "82a6bd694c4f4751a23c370ab51f01a4", "version_major": 2, "version_minor": 0 }, @@ -97,7 +111,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "e3487cd55f6847588492bf7fa51348ca", + "model_id": "3844878631534468a1495e435563e4b0", "version_major": 2, "version_minor": 0 }, @@ -111,9 +125,9 @@ { "data": { "text/plain": [ - "{'sentence': 'ADPnews - Feb 5 , 2010 - Finnish real estate investor Sponda Oyj HEL : SDA1V said today that it slipped to a net loss of EUR 81.5 million USD 11.8 m in 2009 from a profit of EUR 29.3 million in 2008 .',\n", - " 'label': 0,\n", - " 'text_label': 'negative'}" + "{'sentence': 'Finnish elevators and escalators maker KONE Corporation said on Tuesday ( 18 March ) that it has received a major order from Sir Robert McAlpine to supply all elevators and escalators for the Watermark Place project in the City of London .',\n", + " 'label': 2,\n", + " 'text_label': 'positive'}" ] }, "execution_count": 3, @@ -145,39 +159,11 @@ "id": "adf9608c", "metadata": {}, "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "2ce088f4437d4e2c80c267332a5b84e5", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Downloading: 0%| | 0.00/792k [00:00