From a84414f6de63564a9e960ee0f0c58733e016b1d0 Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Fri, 3 Mar 2023 19:36:13 +0530 Subject: [PATCH 1/2] minor fixes to the examples --- .../causal_language_modeling/peft_prefix_tuning_clm.ipynb | 6 +++--- .../causal_language_modeling/peft_prompt_tuning_clm.ipynb | 6 +++--- examples/conditional_generation/peft_lora_seq2seq.ipynb | 6 +++--- .../peft_lora_seq2seq_accelerate_ds_zero3_offload.py | 2 +- .../peft_lora_seq2seq_accelerate_fsdp.py | 4 ++-- .../conditional_generation/peft_prefix_tuning_seq2seq.ipynb | 6 +++--- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/examples/causal_language_modeling/peft_prefix_tuning_clm.ipynb b/examples/causal_language_modeling/peft_prefix_tuning_clm.ipynb index eb25c0c..cf36d21 100644 --- a/examples/causal_language_modeling/peft_prefix_tuning_clm.ipynb +++ b/examples/causal_language_modeling/peft_prefix_tuning_clm.ipynb @@ -1180,9 +1180,9 @@ " tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True)\n", " )\n", "\n", - " eval_epoch_loss = eval_loss / len(train_dataloader)\n", + " eval_epoch_loss = eval_loss / len(eval_dataloader)\n", " eval_ppl = torch.exp(eval_epoch_loss)\n", - " train_epoch_loss = total_loss / len(eval_dataloader)\n", + " train_epoch_loss = total_loss / len(train_dataloader)\n", " train_ppl = torch.exp(train_epoch_loss)\n", " print(f\"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}\")" ] @@ -1345,7 +1345,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 (clang-1300.0.29.30)]" + "version": "3.10.5" }, "vscode": { "interpreter": { diff --git a/examples/causal_language_modeling/peft_prompt_tuning_clm.ipynb b/examples/causal_language_modeling/peft_prompt_tuning_clm.ipynb index e5ba39b..e289206 100644 --- a/examples/causal_language_modeling/peft_prompt_tuning_clm.ipynb +++ b/examples/causal_language_modeling/peft_prompt_tuning_clm.ipynb @@ -1022,9 +1022,9 @@ " tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True)\n", " )\n", "\n", - " eval_epoch_loss = eval_loss / len(train_dataloader)\n", + " eval_epoch_loss = eval_loss / len(eval_dataloader)\n", " eval_ppl = torch.exp(eval_epoch_loss)\n", - " train_epoch_loss = total_loss / len(eval_dataloader)\n", + " train_epoch_loss = total_loss / len(train_dataloader)\n", " train_ppl = torch.exp(train_epoch_loss)\n", " print(f\"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}\")" ] @@ -1185,7 +1185,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 (clang-1300.0.29.30)]" + "version": "3.10.5" }, "vscode": { "interpreter": { diff --git a/examples/conditional_generation/peft_lora_seq2seq.ipynb b/examples/conditional_generation/peft_lora_seq2seq.ipynb index f22d3c6..6cbd4f1 100644 --- a/examples/conditional_generation/peft_lora_seq2seq.ipynb +++ b/examples/conditional_generation/peft_lora_seq2seq.ipynb @@ -324,9 +324,9 @@ " tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True)\n", " )\n", "\n", - " eval_epoch_loss = eval_loss / len(train_dataloader)\n", + " eval_epoch_loss = eval_loss / len(eval_dataloader)\n", " eval_ppl = torch.exp(eval_epoch_loss)\n", - " train_epoch_loss = total_loss / len(eval_dataloader)\n", + " train_epoch_loss = total_loss / len(train_dataloader)\n", " train_ppl = torch.exp(train_epoch_loss)\n", " print(f\"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}\")" ] @@ -473,7 +473,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.10.5" }, "vscode": { "interpreter": { diff --git a/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py b/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py index cef9773..a2d0d20 100644 --- a/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py +++ b/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py @@ -217,7 +217,7 @@ def main(): tracemalloc.cpu_peaked + b2mb(tracemalloc.cpu_begin) ) ) - train_epoch_loss = total_loss / len(eval_dataloader) + train_epoch_loss = total_loss / len(train_dataloader) train_ppl = torch.exp(train_epoch_loss) accelerator.print(f"{epoch=}: {train_ppl=} {train_epoch_loss=}") diff --git a/examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py b/examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py index e00b1ff..c011dbb 100644 --- a/examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py +++ b/examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py @@ -108,9 +108,9 @@ def main(): eval_loss += loss.detach().float() preds = accelerator.gather_for_metrics(torch.argmax(outputs.logits, -1)).detach().cpu().numpy() eval_preds.extend(tokenizer.batch_decode(preds, skip_special_tokens=True)) - eval_epoch_loss = eval_loss / len(train_dataloader) + eval_epoch_loss = eval_loss / len(eval_dataloader) eval_ppl = torch.exp(eval_epoch_loss) - train_epoch_loss = total_loss / len(eval_dataloader) + train_epoch_loss = total_loss / len(train_dataloader) train_ppl = torch.exp(train_epoch_loss) accelerator.print(f"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}") diff --git a/examples/conditional_generation/peft_prefix_tuning_seq2seq.ipynb b/examples/conditional_generation/peft_prefix_tuning_seq2seq.ipynb index dde8fff..aa85f9a 100644 --- a/examples/conditional_generation/peft_prefix_tuning_seq2seq.ipynb +++ b/examples/conditional_generation/peft_prefix_tuning_seq2seq.ipynb @@ -360,9 +360,9 @@ " tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True)\n", " )\n", "\n", - " eval_epoch_loss = eval_loss / len(train_dataloader)\n", + " eval_epoch_loss = eval_loss / len(eval_dataloader)\n", " eval_ppl = torch.exp(eval_epoch_loss)\n", - " train_epoch_loss = total_loss / len(eval_dataloader)\n", + " train_epoch_loss = total_loss / len(train_dataloader)\n", " train_ppl = torch.exp(train_epoch_loss)\n", " print(f\"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}\")" ] @@ -503,7 +503,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 (clang-1300.0.29.30)]" + "version": "3.10.5" }, "vscode": { "interpreter": { From b9451ab458c0fbe15c46b2dbea2a5266e6347b36 Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Tue, 7 Mar 2023 14:04:19 +0530 Subject: [PATCH 2/2] =?UTF-8?q?fixing=20issues=20and=20quality=20=E2=9C=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...ft_lora_clm_accelerate_ds_zero3_offload.py | 21 ++++++---- ...ora_seq2seq_accelerate_ds_zero3_offload.py | 17 +++++--- .../peft_lora_seq2seq_accelerate_fsdp.py | 4 +- examples/lora_dreambooth/train_dreambooth.py | 14 +++---- pyproject.toml | 4 ++ src/peft/peft_model.py | 8 ++-- src/peft/tuners/p_tuning.py | 14 +++---- src/peft/utils/config.py | 5 +-- tests/test_peft_model.py | 42 +++++++++---------- 9 files changed, 73 insertions(+), 56 deletions(-) diff --git a/examples/causal_language_modeling/peft_lora_clm_accelerate_ds_zero3_offload.py b/examples/causal_language_modeling/peft_lora_clm_accelerate_ds_zero3_offload.py index 1f10f1b..daf9d1f 100644 --- a/examples/causal_language_modeling/peft_lora_clm_accelerate_ds_zero3_offload.py +++ b/examples/causal_language_modeling/peft_lora_clm_accelerate_ds_zero3_offload.py @@ -4,9 +4,12 @@ import sys import threading import numpy as np +import psutil import torch from accelerate import Accelerator +from datasets import load_dataset from torch.utils.data import DataLoader +from tqdm import tqdm from transformers import ( AutoModelForCausalLM, AutoTokenizer, @@ -15,10 +18,7 @@ from transformers import ( set_seed, ) -import psutil -from datasets import load_dataset from peft import LoraConfig, TaskType, get_peft_model -from tqdm import tqdm def levenshtein_distance(str1, str2): @@ -280,7 +280,9 @@ def main(): outputs = accelerator.unwrap_model(model).generate( **batch, synced_gpus=is_ds_zero_3, max_new_tokens=10 ) # synced_gpus=True for DS-stage 3 - preds = outputs[:, max_length:].detach().cpu().numpy() + outputs = accelerator.pad_across_processes(outputs, dim=1, pad_index=tokenizer.pad_token_id) + preds = accelerator.gather(outputs) + preds = preds[:, max_length:].detach().cpu().numpy() eval_preds.extend(tokenizer.batch_decode(preds, skip_special_tokens=True)) # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage @@ -304,6 +306,9 @@ def main(): correct = 0 total = 0 + assert len(eval_preds) == len( + dataset["train"][label_column] + ), f"{len(eval_preds)} != {len(dataset['train'][label_column])}" for pred, true in zip(eval_preds, dataset["train"][label_column]): if pred.strip() == true.strip(): correct += 1 @@ -322,15 +327,17 @@ def main(): outputs = accelerator.unwrap_model(model).generate( **batch, synced_gpus=is_ds_zero_3, max_new_tokens=10 ) # synced_gpus=True for DS-stage 3 - test_preds.extend( - tokenizer.batch_decode(outputs[:, max_length:].detach().cpu().numpy(), skip_special_tokens=True) - ) + outputs = accelerator.pad_across_processes(outputs, dim=1, pad_index=tokenizer.pad_token_id) + preds = accelerator.gather(outputs) + preds = preds[:, max_length:].detach().cpu().numpy() + test_preds.extend(tokenizer.batch_decode(preds, skip_special_tokens=True)) test_preds_cleaned = [] for _, pred in enumerate(test_preds): test_preds_cleaned.append(get_closest_label(pred, classes)) test_df = dataset["test"].to_pandas() + assert len(test_preds_cleaned) == len(test_df), f"{len(test_preds_cleaned)} != {len(test_df)}" test_df[label_column] = test_preds_cleaned test_df["text_labels_orig"] = test_preds accelerator.print(test_df[[text_column, label_column]].sample(20)) diff --git a/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py b/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py index a2d0d20..0e47f87 100644 --- a/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py +++ b/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py @@ -4,15 +4,15 @@ import sys import threading import numpy as np +import psutil import torch from accelerate import Accelerator +from datasets import load_dataset from torch.utils.data import DataLoader +from tqdm import tqdm from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, get_linear_schedule_with_warmup, set_seed -import psutil -from datasets import load_dataset from peft import LoraConfig, TaskType, get_peft_model -from tqdm import tqdm def levenshtein_distance(str1, str2): @@ -230,7 +230,8 @@ def main(): outputs = accelerator.unwrap_model(model).generate( **batch, synced_gpus=is_ds_zero_3 ) # synced_gpus=True for DS-stage 3 - preds = outputs.detach().cpu().numpy() + outputs = accelerator.pad_across_processes(outputs, dim=1, pad_index=tokenizer.pad_token_id) + preds = accelerator.gather(outputs).detach().cpu().numpy() eval_preds.extend(tokenizer.batch_decode(preds, skip_special_tokens=True)) # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage @@ -254,6 +255,9 @@ def main(): correct = 0 total = 0 + assert len(eval_preds) == len( + dataset["train"][label_column] + ), f"{len(eval_preds)} != {len(dataset['train'][label_column])}" for pred, true in zip(eval_preds, dataset["train"][label_column]): if pred.strip() == true.strip(): correct += 1 @@ -272,13 +276,16 @@ def main(): outputs = accelerator.unwrap_model(model).generate( **batch, synced_gpus=is_ds_zero_3 ) # synced_gpus=True for DS-stage 3 - test_preds.extend(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True)) + outputs = accelerator.pad_across_processes(outputs, dim=1, pad_index=tokenizer.pad_token_id) + preds = accelerator.gather(outputs).detach().cpu().numpy() + test_preds.extend(tokenizer.batch_decode(preds, skip_special_tokens=True)) test_preds_cleaned = [] for _, pred in enumerate(test_preds): test_preds_cleaned.append(get_closest_label(pred, classes)) test_df = dataset["test"].to_pandas() + assert len(test_preds_cleaned) == len(test_df), f"{len(test_preds_cleaned)} != {len(test_df)}" test_df[label_column] = test_preds_cleaned test_df["text_labels_orig"] = test_preds accelerator.print(test_df[[text_column, label_column]].sample(20)) diff --git a/examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py b/examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py index c011dbb..c2146a5 100644 --- a/examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py +++ b/examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py @@ -2,13 +2,13 @@ import os import torch from accelerate import Accelerator +from datasets import load_dataset from torch.utils.data import DataLoader +from tqdm import tqdm from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, default_data_collator, get_linear_schedule_with_warmup -from datasets import load_dataset from peft import LoraConfig, TaskType, get_peft_model from peft.utils.other import fsdp_auto_wrap_policy -from tqdm import tqdm def main(): diff --git a/examples/lora_dreambooth/train_dreambooth.py b/examples/lora_dreambooth/train_dreambooth.py index 3f1e0cb..c06a175 100644 --- a/examples/lora_dreambooth/train_dreambooth.py +++ b/examples/lora_dreambooth/train_dreambooth.py @@ -11,7 +11,10 @@ import warnings from pathlib import Path from typing import Optional +import datasets +import diffusers import numpy as np +import psutil import torch import torch.nn.functional as F import torch.utils.checkpoint @@ -19,12 +22,6 @@ import transformers from accelerate import Accelerator from accelerate.logging import get_logger from accelerate.utils import set_seed -from torch.utils.data import Dataset -from transformers import AutoTokenizer, PretrainedConfig - -import datasets -import diffusers -import psutil from diffusers import ( AutoencoderKL, DDPMScheduler, @@ -36,10 +33,13 @@ from diffusers.optimization import get_scheduler from diffusers.utils import check_min_version from diffusers.utils.import_utils import is_xformers_available from huggingface_hub import HfFolder, Repository, whoami -from peft import LoraConfig, LoraModel, get_peft_model_state_dict from PIL import Image +from torch.utils.data import Dataset from torchvision import transforms from tqdm.auto import tqdm +from transformers import AutoTokenizer, PretrainedConfig + +from peft import LoraConfig, LoraModel, get_peft_model_state_dict # Will error if the minimal version of diffusers is not installed. Remove at your own risks. diff --git a/pyproject.toml b/pyproject.toml index a135358..a72149d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,10 @@ select = ["C", "E", "F", "I", "W"] line-length = 119 [tool.ruff.isort] +lines-after-imports = 2 +known-first-party = ["peft"] + +[isort] default_section = "FIRSTPARTY" known_first_party = "peft" known_third_party = [ diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py index 4703059..e33da41 100644 --- a/src/peft/peft_model.py +++ b/src/peft/peft_model.py @@ -22,13 +22,12 @@ import torch from accelerate import dispatch_model, infer_auto_device_map from accelerate.hooks import AlignDevicesHook, add_hook_to_module, remove_hook_from_submodules from accelerate.utils import get_balanced_memory +from huggingface_hub import hf_hub_download from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from transformers import PreTrainedModel from transformers.modeling_outputs import SequenceClassifierOutput, TokenClassifierOutput from transformers.utils import PushToHubMixin -from huggingface_hub import hf_hub_download - from .tuners import LoraModel, PrefixEncoder, PromptEmbedding, PromptEncoder from .utils import ( TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPING, @@ -156,7 +155,8 @@ class PeftModel(PushToHubMixin, torch.nn.Module): ) adapters_weights = torch.load( - filename, map_location=torch.device("cuda" if torch.cuda.is_available() else "cpu")) + filename, map_location=torch.device("cuda" if torch.cuda.is_available() else "cpu") + ) # load the weights into the model model = set_peft_model_state_dict(model, adapters_weights) if getattr(model, "hf_device_map", None) is not None: @@ -271,7 +271,7 @@ class PeftModel(PushToHubMixin, torch.nn.Module): # if using DS Zero 3 and the weights are initialized empty if num_params == 0 and hasattr(param, "ds_numel"): num_params = param.ds_numel - + all_param += num_params if param.requires_grad: trainable_params += param.numel() diff --git a/src/peft/tuners/p_tuning.py b/src/peft/tuners/p_tuning.py index 31905af..b9c38c4 100644 --- a/src/peft/tuners/p_tuning.py +++ b/src/peft/tuners/p_tuning.py @@ -14,6 +14,7 @@ # limitations under the License. import enum +import warnings from dataclasses import dataclass, field from typing import Union @@ -131,17 +132,16 @@ class PromptEncoder(torch.nn.Module): ) elif self.encoder_type == PromptEncoderReparameterizationType.MLP: + warnings.warn( + f"for {self.encoder_type}, the `encoder_num_layers` is ignored. Exactly 2 MLP layers are used." + ) layers = [ torch.nn.Linear(self.input_size, self.hidden_size), torch.nn.ReLU(), + torch.nn.Linear(self.hidden_size, self.hidden_size), + torch.nn.ReLU(), + torch.nn.Linear(self.hidden_size, self.output_size), ] - layers.extend( - [ - torch.nn.Linear(self.hidden_size, self.hidden_size), - torch.nn.ReLU(), - ] - ) - layers.append(torch.nn.Linear(self.hidden_size, self.output_size)) self.mlp_head = torch.nn.Sequential(*layers) else: diff --git a/src/peft/utils/config.py b/src/peft/utils/config.py index f0587fe..611e452 100644 --- a/src/peft/utils/config.py +++ b/src/peft/utils/config.py @@ -18,9 +18,8 @@ import os from dataclasses import asdict, dataclass, field from typing import Optional, Union -from transformers.utils import PushToHubMixin - from huggingface_hub import hf_hub_download +from transformers.utils import PushToHubMixin from .adapters_utils import CONFIG_NAME @@ -98,7 +97,7 @@ class PeftConfigMixin(PushToHubMixin): else: try: config_file = hf_hub_download(pretrained_model_name_or_path, CONFIG_NAME) - except: + except Exception: raise ValueError(f"Can't find config.json at '{pretrained_model_name_or_path}'") loaded_attributes = cls.from_json_file(config_file) diff --git a/tests/test_peft_model.py b/tests/test_peft_model.py index 5ceda03..0e85c7c 100644 --- a/tests/test_peft_model.py +++ b/tests/test_peft_model.py @@ -42,27 +42,27 @@ class PeftTestMixin: PromptTuningConfig, ) config_kwargs = ( - dict( - r=8, - lora_alpha=32, - target_modules=["q_proj", "v_proj"], - lora_dropout=0.05, - bias="none", - task_type="CAUSAL_LM", - ), - dict( - num_virtual_tokens=10, - task_type="CAUSAL_LM", - ), - dict( - num_virtual_tokens=10, - encoder_hidden_size=32, - task_type="CAUSAL_LM", - ), - dict( - num_virtual_tokens=10, - task_type="CAUSAL_LM", - ), + { + "r": 8, + "lora_alpha": 32, + "target_modules": ["q_proj", "v_proj"], + "lora_dropout": 0.05, + "bias": "none", + "task_type": "CAUSAL_LM", + }, + { + "num_virtual_tokens": 10, + "task_type": "CAUSAL_LM", + }, + { + "num_virtual_tokens": 10, + "encoder_hidden_size": 32, + "task_type": "CAUSAL_LM", + }, + { + "num_virtual_tokens": 10, + "task_type": "CAUSAL_LM", + }, )