From 6175ee2c4cef03988cc3c7f78f99fc1a3431cf94 Mon Sep 17 00:00:00 2001 From: Saurav Maheshkar Date: Thu, 23 Feb 2023 11:30:14 +0000 Subject: [PATCH 01/28] chore: drop MANIFEST --- MANIFEST.in | 1 - setup.py | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 1aba38f..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include LICENSE diff --git a/setup.py b/setup.py index b51aa93..4428228 100644 --- a/setup.py +++ b/setup.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from setuptools import setup -from setuptools import find_packages +from setuptools import find_packages, setup + extras = {} extras["quality"] = ["black ~= 22.0", "isort >= 5.5.4", "flake8 >= 3.8.3"] @@ -24,6 +24,7 @@ setup( name="peft", version="0.2.0.dev0", description="Parameter-Efficient Fine-Tuning (PEFT)", + license_files=["LICENSE"], long_description=open("README.md", "r", encoding="utf-8").read(), long_description_content_type="text/markdown", keywords="deep learning", From a78f8a0495c5d8e516912ae4be77e3a4af2e4ea8 Mon Sep 17 00:00:00 2001 From: Saurav Maheshkar Date: Thu, 23 Feb 2023 11:34:18 +0000 Subject: [PATCH 02/28] style: move isort and pytest config to pyproject --- pyproject.toml | 24 ++++++++++++++++++++++++ setup.cfg | 20 -------------------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b7465bb..1947fa5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,27 @@ [tool.black] line-length = 119 target-version = ['py36'] + +[tool.isort] +line_length = 119 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true +lines_after_imports = 2 +default_section = "FIRSTPARTY" +known_first_party = "pet" +known_third_party = [ + "numpy", + "torch", + "accelerate", + "transformers", +] + +[tool.pytest.ini_options] +doctest_optionflags = [ + "NUMBER", + "NORMALIZE_WHITESPACE", + "ELLIPSIS", +] diff --git a/setup.cfg b/setup.cfg index 6b26312..ab1382c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,23 +1,3 @@ -[isort] -default_section = FIRSTPARTY -ensure_newline_before_comments = True -force_grid_wrap = 0 -include_trailing_comma = True -known_first_party = pet -known_third_party = - numpy - torch - accelerate - transformers - -line_length = 119 -lines_after_imports = 2 -multi_line_output = 3 -use_parentheses = True - [flake8] ignore = E203, E722, E501, E741, W503, W605 max-line-length = 119 - -[tool:pytest] -doctest_optionflags=NUMBER NORMALIZE_WHITESPACE ELLIPSIS \ No newline at end of file From 99901896ccea794fd012b81fbf747c3476d98388 Mon Sep 17 00:00:00 2001 From: Saurav Maheshkar Date: Mon, 27 Feb 2023 10:50:10 +0000 Subject: [PATCH 03/28] style: switch to ruff --- pyproject.toml | 27 ++++++++++++++++----------- setup.py | 3 +-- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1947fa5..34ea10c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,14 +2,12 @@ line-length = 119 target-version = ['py36'] -[tool.isort] -line_length = 119 -multi_line_output = 3 -include_trailing_comma = true -force_grid_wrap = 0 -use_parentheses = true -ensure_newline_before_comments = true -lines_after_imports = 2 +[tool.ruff] +ignore = ["C901", "E501", "E741", "W605"] +select = ["C", "E", "F", "I", "W"] +line-length = 119 + +[tool.ruff.isort] default_section = "FIRSTPARTY" known_first_party = "pet" known_third_party = [ @@ -18,10 +16,17 @@ known_third_party = [ "accelerate", "transformers", ] +line_length = 119 +lines_after_imports = 2 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true -[tool.pytest.ini_options] +[tool.pytest] doctest_optionflags = [ - "NUMBER", "NORMALIZE_WHITESPACE", "ELLIPSIS", -] + "NUMBER", +] \ No newline at end of file diff --git a/setup.py b/setup.py index 4428228..6b26477 100644 --- a/setup.py +++ b/setup.py @@ -14,9 +14,8 @@ from setuptools import find_packages, setup - extras = {} -extras["quality"] = ["black ~= 22.0", "isort >= 5.5.4", "flake8 >= 3.8.3"] +extras["quality"] = ["black ~= 22.0", "ruff>=0.0.241"] extras["docs_specific"] = ["hf-doc-builder"] extras["dev"] = extras["quality"] + extras["docs_specific"] From 47601bab7c4e41899e4ca135e5922c4863182e34 Mon Sep 17 00:00:00 2001 From: Saurav Maheshkar Date: Tue, 28 Feb 2023 02:58:12 +0530 Subject: [PATCH 04/28] chore: drop `setup.cfg` as we shifted to `ruff` --- setup.cfg | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index ab1382c..0000000 --- a/setup.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[flake8] -ignore = E203, E722, E501, E741, W503, W605 -max-line-length = 119 From 7820a539dd712788efd7ef107338ea3b1279ec7a Mon Sep 17 00:00:00 2001 From: Saurav Maheshkar Date: Tue, 28 Feb 2023 16:10:38 +0530 Subject: [PATCH 05/28] fix(pyproject.toml): update `known_first_party` Co-authored-by: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 34ea10c..a135358 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ line-length = 119 [tool.ruff.isort] default_section = "FIRSTPARTY" -known_first_party = "pet" +known_first_party = "peft" known_third_party = [ "numpy", "torch", From 94f00b7d272932ba8f16e6b3ec485eefe900bc70 Mon Sep 17 00:00:00 2001 From: Saurav Maheshkar Date: Tue, 28 Feb 2023 10:46:07 +0000 Subject: [PATCH 06/28] chore: update Makefile with ruff commands --- Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 8d8148e..ff8ed42 100644 --- a/Makefile +++ b/Makefile @@ -7,13 +7,12 @@ check_dirs := src tests examples # this target runs checks on all files quality: black --check $(check_dirs) - isort --check-only $(check_dirs) - flake8 $(check_dirs) + ruff $(check_dirs) doc-builder style src tests --max_len 119 --check_only # Format source code automatically and check is there are any problems left that need manual fixing style: black $(check_dirs) - isort $(check_dirs) + ruff $(check_dirs) --fix doc-builder style src tests --max_len 119 \ No newline at end of file From a84414f6de63564a9e960ee0f0c58733e016b1d0 Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Fri, 3 Mar 2023 19:36:13 +0530 Subject: [PATCH 07/28] minor fixes to the examples --- .../causal_language_modeling/peft_prefix_tuning_clm.ipynb | 6 +++--- .../causal_language_modeling/peft_prompt_tuning_clm.ipynb | 6 +++--- examples/conditional_generation/peft_lora_seq2seq.ipynb | 6 +++--- .../peft_lora_seq2seq_accelerate_ds_zero3_offload.py | 2 +- .../peft_lora_seq2seq_accelerate_fsdp.py | 4 ++-- .../conditional_generation/peft_prefix_tuning_seq2seq.ipynb | 6 +++--- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/examples/causal_language_modeling/peft_prefix_tuning_clm.ipynb b/examples/causal_language_modeling/peft_prefix_tuning_clm.ipynb index eb25c0c..cf36d21 100644 --- a/examples/causal_language_modeling/peft_prefix_tuning_clm.ipynb +++ b/examples/causal_language_modeling/peft_prefix_tuning_clm.ipynb @@ -1180,9 +1180,9 @@ " tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True)\n", " )\n", "\n", - " eval_epoch_loss = eval_loss / len(train_dataloader)\n", + " eval_epoch_loss = eval_loss / len(eval_dataloader)\n", " eval_ppl = torch.exp(eval_epoch_loss)\n", - " train_epoch_loss = total_loss / len(eval_dataloader)\n", + " train_epoch_loss = total_loss / len(train_dataloader)\n", " train_ppl = torch.exp(train_epoch_loss)\n", " print(f\"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}\")" ] @@ -1345,7 +1345,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 (clang-1300.0.29.30)]" + "version": "3.10.5" }, "vscode": { "interpreter": { diff --git a/examples/causal_language_modeling/peft_prompt_tuning_clm.ipynb b/examples/causal_language_modeling/peft_prompt_tuning_clm.ipynb index e5ba39b..e289206 100644 --- a/examples/causal_language_modeling/peft_prompt_tuning_clm.ipynb +++ b/examples/causal_language_modeling/peft_prompt_tuning_clm.ipynb @@ -1022,9 +1022,9 @@ " tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True)\n", " )\n", "\n", - " eval_epoch_loss = eval_loss / len(train_dataloader)\n", + " eval_epoch_loss = eval_loss / len(eval_dataloader)\n", " eval_ppl = torch.exp(eval_epoch_loss)\n", - " train_epoch_loss = total_loss / len(eval_dataloader)\n", + " train_epoch_loss = total_loss / len(train_dataloader)\n", " train_ppl = torch.exp(train_epoch_loss)\n", " print(f\"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}\")" ] @@ -1185,7 +1185,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 (clang-1300.0.29.30)]" + "version": "3.10.5" }, "vscode": { "interpreter": { diff --git a/examples/conditional_generation/peft_lora_seq2seq.ipynb b/examples/conditional_generation/peft_lora_seq2seq.ipynb index f22d3c6..6cbd4f1 100644 --- a/examples/conditional_generation/peft_lora_seq2seq.ipynb +++ b/examples/conditional_generation/peft_lora_seq2seq.ipynb @@ -324,9 +324,9 @@ " tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True)\n", " )\n", "\n", - " eval_epoch_loss = eval_loss / len(train_dataloader)\n", + " eval_epoch_loss = eval_loss / len(eval_dataloader)\n", " eval_ppl = torch.exp(eval_epoch_loss)\n", - " train_epoch_loss = total_loss / len(eval_dataloader)\n", + " train_epoch_loss = total_loss / len(train_dataloader)\n", " train_ppl = torch.exp(train_epoch_loss)\n", " print(f\"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}\")" ] @@ -473,7 +473,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.10.5" }, "vscode": { "interpreter": { diff --git a/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py b/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py index cef9773..a2d0d20 100644 --- a/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py +++ b/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py @@ -217,7 +217,7 @@ def main(): tracemalloc.cpu_peaked + b2mb(tracemalloc.cpu_begin) ) ) - train_epoch_loss = total_loss / len(eval_dataloader) + train_epoch_loss = total_loss / len(train_dataloader) train_ppl = torch.exp(train_epoch_loss) accelerator.print(f"{epoch=}: {train_ppl=} {train_epoch_loss=}") diff --git a/examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py b/examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py index e00b1ff..c011dbb 100644 --- a/examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py +++ b/examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py @@ -108,9 +108,9 @@ def main(): eval_loss += loss.detach().float() preds = accelerator.gather_for_metrics(torch.argmax(outputs.logits, -1)).detach().cpu().numpy() eval_preds.extend(tokenizer.batch_decode(preds, skip_special_tokens=True)) - eval_epoch_loss = eval_loss / len(train_dataloader) + eval_epoch_loss = eval_loss / len(eval_dataloader) eval_ppl = torch.exp(eval_epoch_loss) - train_epoch_loss = total_loss / len(eval_dataloader) + train_epoch_loss = total_loss / len(train_dataloader) train_ppl = torch.exp(train_epoch_loss) accelerator.print(f"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}") diff --git a/examples/conditional_generation/peft_prefix_tuning_seq2seq.ipynb b/examples/conditional_generation/peft_prefix_tuning_seq2seq.ipynb index dde8fff..aa85f9a 100644 --- a/examples/conditional_generation/peft_prefix_tuning_seq2seq.ipynb +++ b/examples/conditional_generation/peft_prefix_tuning_seq2seq.ipynb @@ -360,9 +360,9 @@ " tokenizer.batch_decode(torch.argmax(outputs.logits, -1).detach().cpu().numpy(), skip_special_tokens=True)\n", " )\n", "\n", - " eval_epoch_loss = eval_loss / len(train_dataloader)\n", + " eval_epoch_loss = eval_loss / len(eval_dataloader)\n", " eval_ppl = torch.exp(eval_epoch_loss)\n", - " train_epoch_loss = total_loss / len(eval_dataloader)\n", + " train_epoch_loss = total_loss / len(train_dataloader)\n", " train_ppl = torch.exp(train_epoch_loss)\n", " print(f\"{epoch=}: {train_ppl=} {train_epoch_loss=} {eval_ppl=} {eval_epoch_loss=}\")" ] @@ -503,7 +503,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.5 (v3.10.5:f377153967, Jun 6 2022, 12:36:10) [Clang 13.0.0 (clang-1300.0.29.30)]" + "version": "3.10.5" }, "vscode": { "interpreter": { From 53eb20938767ad88c8c878668124a7427bd5c489 Mon Sep 17 00:00:00 2001 From: mayank31398 Date: Fri, 3 Mar 2023 23:43:25 +0530 Subject: [PATCH 08/28] support option for encoder only prompts --- src/peft/peft_model.py | 25 ++++++++++++++++--------- src/peft/utils/config.py | 2 +- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py index 4703059..b0c9bac 100644 --- a/src/peft/peft_model.py +++ b/src/peft/peft_model.py @@ -184,7 +184,6 @@ class PeftModel(PushToHubMixin, torch.nn.Module): return model def _setup_prompt_encoder(self): - num_transformer_submodules = 0 transformer_backbone = None for name, module in self.base_model.named_children(): for param in module.parameters(): @@ -194,8 +193,9 @@ class PeftModel(PushToHubMixin, torch.nn.Module): if transformer_backbone is None: transformer_backbone = module self.transformer_backbone_name = name - num_transformer_submodules += 1 - self.peft_config.num_transformer_submodules = 2 if self.peft_config.task_type == TaskType.SEQ_2_SEQ_LM else 1 + + if self.peft_config.num_transformer_submodules is None: + self.peft_config.num_transformer_submodules = 2 if self.peft_config.task_type == TaskType.SEQ_2_SEQ_LM else 1 for named_param, value in list(transformer_backbone.named_parameters()): if value.shape[0] == self.base_model.config.vocab_size: @@ -719,15 +719,22 @@ class PeftModelForSeq2SeqLM(PeftModel): kwargs["attention_mask"] = torch.cat((prefix_attention_mask, attention_mask), dim=1) # concat prompt labels if labels is not None: - prefix_labels = torch.full((batch_size, self.peft_config.num_virtual_tokens), -100).to(self.device) - kwargs["labels"] = torch.cat((prefix_labels, labels), dim=1) + if self.peft_config.num_transformer_submodules == 1: + kwargs["labels"] = labels + elif self.peft_config.num_transformer_submodules == 2: + prefix_labels = torch.full((batch_size, self.peft_config.num_virtual_tokens), -100).to(self.device) + kwargs["labels"] = torch.cat((prefix_labels, labels), dim=1) prompts = self.get_prompt(batch_size=batch_size) prompts = prompts.to(inputs_embeds.dtype) inputs_embeds = torch.cat((prompts[:, : self.peft_config.num_virtual_tokens], inputs_embeds), dim=1) - decoder_inputs_embeds = torch.cat( - (prompts[:, self.peft_config.num_virtual_tokens :], decoder_inputs_embeds), dim=1 - ) - return self.base_model(inputs_embeds=inputs_embeds, decoder_inputs_embeds=decoder_inputs_embeds, **kwargs) + if self.peft_config.num_transformer_submodules == 1: + return self.base_model(inputs_embeds=inputs_embeds, **kwargs) + elif self.peft_config.num_transformer_submodules == 2: + decoder_inputs_embeds = torch.cat( + (prompts[:, self.peft_config.num_virtual_tokens :], decoder_inputs_embeds), dim=1 + ) + return self.base_model(inputs_embeds=inputs_embeds, decoder_inputs_embeds=decoder_inputs_embeds, **kwargs) + def generate(self, **kwargs): if not isinstance(self.peft_config, PromptLearningConfig): diff --git a/src/peft/utils/config.py b/src/peft/utils/config.py index f0587fe..c0a55f5 100644 --- a/src/peft/utils/config.py +++ b/src/peft/utils/config.py @@ -161,6 +161,6 @@ class PromptLearningConfig(PeftConfig): token_dim: int = field( default=None, metadata={"help": "The hidden embedding dimension of the base transformer model"} ) - num_transformer_submodules: Optional[int] = field(default=1, metadata={"help": "Number of transformer submodules"}) + num_transformer_submodules: Optional[int] = field(default=None, metadata={"help": "Number of transformer submodules"}) num_attention_heads: Optional[int] = field(default=None, metadata={"help": "Number of attention heads"}) num_layers: Optional[int] = field(default=None, metadata={"help": "Number of transformer layers"}) From b9451ab458c0fbe15c46b2dbea2a5266e6347b36 Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Tue, 7 Mar 2023 14:04:19 +0530 Subject: [PATCH 09/28] =?UTF-8?q?fixing=20issues=20and=20quality=20?= =?UTF-8?q?=E2=9C=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...ft_lora_clm_accelerate_ds_zero3_offload.py | 21 ++++++---- ...ora_seq2seq_accelerate_ds_zero3_offload.py | 17 +++++--- .../peft_lora_seq2seq_accelerate_fsdp.py | 4 +- examples/lora_dreambooth/train_dreambooth.py | 14 +++---- pyproject.toml | 4 ++ src/peft/peft_model.py | 8 ++-- src/peft/tuners/p_tuning.py | 14 +++---- src/peft/utils/config.py | 5 +-- tests/test_peft_model.py | 42 +++++++++---------- 9 files changed, 73 insertions(+), 56 deletions(-) diff --git a/examples/causal_language_modeling/peft_lora_clm_accelerate_ds_zero3_offload.py b/examples/causal_language_modeling/peft_lora_clm_accelerate_ds_zero3_offload.py index 1f10f1b..daf9d1f 100644 --- a/examples/causal_language_modeling/peft_lora_clm_accelerate_ds_zero3_offload.py +++ b/examples/causal_language_modeling/peft_lora_clm_accelerate_ds_zero3_offload.py @@ -4,9 +4,12 @@ import sys import threading import numpy as np +import psutil import torch from accelerate import Accelerator +from datasets import load_dataset from torch.utils.data import DataLoader +from tqdm import tqdm from transformers import ( AutoModelForCausalLM, AutoTokenizer, @@ -15,10 +18,7 @@ from transformers import ( set_seed, ) -import psutil -from datasets import load_dataset from peft import LoraConfig, TaskType, get_peft_model -from tqdm import tqdm def levenshtein_distance(str1, str2): @@ -280,7 +280,9 @@ def main(): outputs = accelerator.unwrap_model(model).generate( **batch, synced_gpus=is_ds_zero_3, max_new_tokens=10 ) # synced_gpus=True for DS-stage 3 - preds = outputs[:, max_length:].detach().cpu().numpy() + outputs = accelerator.pad_across_processes(outputs, dim=1, pad_index=tokenizer.pad_token_id) + preds = accelerator.gather(outputs) + preds = preds[:, max_length:].detach().cpu().numpy() eval_preds.extend(tokenizer.batch_decode(preds, skip_special_tokens=True)) # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage @@ -304,6 +306,9 @@ def main(): correct = 0 total = 0 + assert len(eval_preds) == len( + dataset["train"][label_column] + ), f"{len(eval_preds)} != {len(dataset['train'][label_column])}" for pred, true in zip(eval_preds, dataset["train"][label_column]): if pred.strip() == true.strip(): correct += 1 @@ -322,15 +327,17 @@ def main(): outputs = accelerator.unwrap_model(model).generate( **batch, synced_gpus=is_ds_zero_3, max_new_tokens=10 ) # synced_gpus=True for DS-stage 3 - test_preds.extend( - tokenizer.batch_decode(outputs[:, max_length:].detach().cpu().numpy(), skip_special_tokens=True) - ) + outputs = accelerator.pad_across_processes(outputs, dim=1, pad_index=tokenizer.pad_token_id) + preds = accelerator.gather(outputs) + preds = preds[:, max_length:].detach().cpu().numpy() + test_preds.extend(tokenizer.batch_decode(preds, skip_special_tokens=True)) test_preds_cleaned = [] for _, pred in enumerate(test_preds): test_preds_cleaned.append(get_closest_label(pred, classes)) test_df = dataset["test"].to_pandas() + assert len(test_preds_cleaned) == len(test_df), f"{len(test_preds_cleaned)} != {len(test_df)}" test_df[label_column] = test_preds_cleaned test_df["text_labels_orig"] = test_preds accelerator.print(test_df[[text_column, label_column]].sample(20)) diff --git a/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py b/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py index a2d0d20..0e47f87 100644 --- a/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py +++ b/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py @@ -4,15 +4,15 @@ import sys import threading import numpy as np +import psutil import torch from accelerate import Accelerator +from datasets import load_dataset from torch.utils.data import DataLoader +from tqdm import tqdm from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, get_linear_schedule_with_warmup, set_seed -import psutil -from datasets import load_dataset from peft import LoraConfig, TaskType, get_peft_model -from tqdm import tqdm def levenshtein_distance(str1, str2): @@ -230,7 +230,8 @@ def main(): outputs = accelerator.unwrap_model(model).generate( **batch, synced_gpus=is_ds_zero_3 ) # synced_gpus=True for DS-stage 3 - preds = outputs.detach().cpu().numpy() + outputs = accelerator.pad_across_processes(outputs, dim=1, pad_index=tokenizer.pad_token_id) + preds = accelerator.gather(outputs).detach().cpu().numpy() eval_preds.extend(tokenizer.batch_decode(preds, skip_special_tokens=True)) # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage @@ -254,6 +255,9 @@ def main(): correct = 0 total = 0 + assert len(eval_preds) == len( + dataset["train"][label_column] + ), f"{len(eval_preds)} != {len(dataset['train'][label_column])}" for pred, true in zip(eval_preds, dataset["train"][label_column]): if pred.strip() == true.strip(): correct += 1 @@ -272,13 +276,16 @@ def main(): outputs = accelerator.unwrap_model(model).generate( **batch, synced_gpus=is_ds_zero_3 ) # synced_gpus=True for DS-stage 3 - test_preds.extend(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True)) + outputs = accelerator.pad_across_processes(outputs, dim=1, pad_index=tokenizer.pad_token_id) + preds = accelerator.gather(outputs).detach().cpu().numpy() + test_preds.extend(tokenizer.batch_decode(preds, skip_special_tokens=True)) test_preds_cleaned = [] for _, pred in enumerate(test_preds): test_preds_cleaned.append(get_closest_label(pred, classes)) test_df = dataset["test"].to_pandas() + assert len(test_preds_cleaned) == len(test_df), f"{len(test_preds_cleaned)} != {len(test_df)}" test_df[label_column] = test_preds_cleaned test_df["text_labels_orig"] = test_preds accelerator.print(test_df[[text_column, label_column]].sample(20)) diff --git a/examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py b/examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py index c011dbb..c2146a5 100644 --- a/examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py +++ b/examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py @@ -2,13 +2,13 @@ import os import torch from accelerate import Accelerator +from datasets import load_dataset from torch.utils.data import DataLoader +from tqdm import tqdm from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, default_data_collator, get_linear_schedule_with_warmup -from datasets import load_dataset from peft import LoraConfig, TaskType, get_peft_model from peft.utils.other import fsdp_auto_wrap_policy -from tqdm import tqdm def main(): diff --git a/examples/lora_dreambooth/train_dreambooth.py b/examples/lora_dreambooth/train_dreambooth.py index 3f1e0cb..c06a175 100644 --- a/examples/lora_dreambooth/train_dreambooth.py +++ b/examples/lora_dreambooth/train_dreambooth.py @@ -11,7 +11,10 @@ import warnings from pathlib import Path from typing import Optional +import datasets +import diffusers import numpy as np +import psutil import torch import torch.nn.functional as F import torch.utils.checkpoint @@ -19,12 +22,6 @@ import transformers from accelerate import Accelerator from accelerate.logging import get_logger from accelerate.utils import set_seed -from torch.utils.data import Dataset -from transformers import AutoTokenizer, PretrainedConfig - -import datasets -import diffusers -import psutil from diffusers import ( AutoencoderKL, DDPMScheduler, @@ -36,10 +33,13 @@ from diffusers.optimization import get_scheduler from diffusers.utils import check_min_version from diffusers.utils.import_utils import is_xformers_available from huggingface_hub import HfFolder, Repository, whoami -from peft import LoraConfig, LoraModel, get_peft_model_state_dict from PIL import Image +from torch.utils.data import Dataset from torchvision import transforms from tqdm.auto import tqdm +from transformers import AutoTokenizer, PretrainedConfig + +from peft import LoraConfig, LoraModel, get_peft_model_state_dict # Will error if the minimal version of diffusers is not installed. Remove at your own risks. diff --git a/pyproject.toml b/pyproject.toml index a135358..a72149d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,10 @@ select = ["C", "E", "F", "I", "W"] line-length = 119 [tool.ruff.isort] +lines-after-imports = 2 +known-first-party = ["peft"] + +[isort] default_section = "FIRSTPARTY" known_first_party = "peft" known_third_party = [ diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py index 4703059..e33da41 100644 --- a/src/peft/peft_model.py +++ b/src/peft/peft_model.py @@ -22,13 +22,12 @@ import torch from accelerate import dispatch_model, infer_auto_device_map from accelerate.hooks import AlignDevicesHook, add_hook_to_module, remove_hook_from_submodules from accelerate.utils import get_balanced_memory +from huggingface_hub import hf_hub_download from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from transformers import PreTrainedModel from transformers.modeling_outputs import SequenceClassifierOutput, TokenClassifierOutput from transformers.utils import PushToHubMixin -from huggingface_hub import hf_hub_download - from .tuners import LoraModel, PrefixEncoder, PromptEmbedding, PromptEncoder from .utils import ( TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPING, @@ -156,7 +155,8 @@ class PeftModel(PushToHubMixin, torch.nn.Module): ) adapters_weights = torch.load( - filename, map_location=torch.device("cuda" if torch.cuda.is_available() else "cpu")) + filename, map_location=torch.device("cuda" if torch.cuda.is_available() else "cpu") + ) # load the weights into the model model = set_peft_model_state_dict(model, adapters_weights) if getattr(model, "hf_device_map", None) is not None: @@ -271,7 +271,7 @@ class PeftModel(PushToHubMixin, torch.nn.Module): # if using DS Zero 3 and the weights are initialized empty if num_params == 0 and hasattr(param, "ds_numel"): num_params = param.ds_numel - + all_param += num_params if param.requires_grad: trainable_params += param.numel() diff --git a/src/peft/tuners/p_tuning.py b/src/peft/tuners/p_tuning.py index 31905af..b9c38c4 100644 --- a/src/peft/tuners/p_tuning.py +++ b/src/peft/tuners/p_tuning.py @@ -14,6 +14,7 @@ # limitations under the License. import enum +import warnings from dataclasses import dataclass, field from typing import Union @@ -131,17 +132,16 @@ class PromptEncoder(torch.nn.Module): ) elif self.encoder_type == PromptEncoderReparameterizationType.MLP: + warnings.warn( + f"for {self.encoder_type}, the `encoder_num_layers` is ignored. Exactly 2 MLP layers are used." + ) layers = [ torch.nn.Linear(self.input_size, self.hidden_size), torch.nn.ReLU(), + torch.nn.Linear(self.hidden_size, self.hidden_size), + torch.nn.ReLU(), + torch.nn.Linear(self.hidden_size, self.output_size), ] - layers.extend( - [ - torch.nn.Linear(self.hidden_size, self.hidden_size), - torch.nn.ReLU(), - ] - ) - layers.append(torch.nn.Linear(self.hidden_size, self.output_size)) self.mlp_head = torch.nn.Sequential(*layers) else: diff --git a/src/peft/utils/config.py b/src/peft/utils/config.py index f0587fe..611e452 100644 --- a/src/peft/utils/config.py +++ b/src/peft/utils/config.py @@ -18,9 +18,8 @@ import os from dataclasses import asdict, dataclass, field from typing import Optional, Union -from transformers.utils import PushToHubMixin - from huggingface_hub import hf_hub_download +from transformers.utils import PushToHubMixin from .adapters_utils import CONFIG_NAME @@ -98,7 +97,7 @@ class PeftConfigMixin(PushToHubMixin): else: try: config_file = hf_hub_download(pretrained_model_name_or_path, CONFIG_NAME) - except: + except Exception: raise ValueError(f"Can't find config.json at '{pretrained_model_name_or_path}'") loaded_attributes = cls.from_json_file(config_file) diff --git a/tests/test_peft_model.py b/tests/test_peft_model.py index 5ceda03..0e85c7c 100644 --- a/tests/test_peft_model.py +++ b/tests/test_peft_model.py @@ -42,27 +42,27 @@ class PeftTestMixin: PromptTuningConfig, ) config_kwargs = ( - dict( - r=8, - lora_alpha=32, - target_modules=["q_proj", "v_proj"], - lora_dropout=0.05, - bias="none", - task_type="CAUSAL_LM", - ), - dict( - num_virtual_tokens=10, - task_type="CAUSAL_LM", - ), - dict( - num_virtual_tokens=10, - encoder_hidden_size=32, - task_type="CAUSAL_LM", - ), - dict( - num_virtual_tokens=10, - task_type="CAUSAL_LM", - ), + { + "r": 8, + "lora_alpha": 32, + "target_modules": ["q_proj", "v_proj"], + "lora_dropout": 0.05, + "bias": "none", + "task_type": "CAUSAL_LM", + }, + { + "num_virtual_tokens": 10, + "task_type": "CAUSAL_LM", + }, + { + "num_virtual_tokens": 10, + "encoder_hidden_size": 32, + "task_type": "CAUSAL_LM", + }, + { + "num_virtual_tokens": 10, + "task_type": "CAUSAL_LM", + }, ) From c81b6680e7b9d49ab122b2d471f902776c4d97ca Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Tue, 7 Mar 2023 17:59:02 +0530 Subject: [PATCH 10/28] adding 8bitMegredLinear lora --- src/peft/peft_model.py | 9 +++-- src/peft/tuners/lora.py | 87 +++++++++++++++++++++++++++++++++++++++- src/peft/utils/config.py | 4 +- 3 files changed, 94 insertions(+), 6 deletions(-) diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py index b885624..ed33ba7 100644 --- a/src/peft/peft_model.py +++ b/src/peft/peft_model.py @@ -195,7 +195,9 @@ class PeftModel(PushToHubMixin, torch.nn.Module): self.transformer_backbone_name = name if self.peft_config.num_transformer_submodules is None: - self.peft_config.num_transformer_submodules = 2 if self.peft_config.task_type == TaskType.SEQ_2_SEQ_LM else 1 + self.peft_config.num_transformer_submodules = ( + 2 if self.peft_config.task_type == TaskType.SEQ_2_SEQ_LM else 1 + ) for named_param, value in list(transformer_backbone.named_parameters()): if value.shape[0] == self.base_model.config.vocab_size: @@ -733,8 +735,9 @@ class PeftModelForSeq2SeqLM(PeftModel): decoder_inputs_embeds = torch.cat( (prompts[:, self.peft_config.num_virtual_tokens :], decoder_inputs_embeds), dim=1 ) - return self.base_model(inputs_embeds=inputs_embeds, decoder_inputs_embeds=decoder_inputs_embeds, **kwargs) - + return self.base_model( + inputs_embeds=inputs_embeds, decoder_inputs_embeds=decoder_inputs_embeds, **kwargs + ) def generate(self, **kwargs): if not isinstance(self.peft_config, PromptLearningConfig): diff --git a/src/peft/tuners/lora.py b/src/peft/tuners/lora.py index 6756b0c..ed10675 100644 --- a/src/peft/tuners/lora.py +++ b/src/peft/tuners/lora.py @@ -145,7 +145,7 @@ class LoraModel(torch.nn.Module): is_target_modules_in_base_model = True parent, target, target_name = self._get_submodules(key) bias = target.bias is not None - if loaded_in_8bit and isinstance(target, bnb.nn.Linear8bitLt) and self.peft_config.enable_lora is None: + if loaded_in_8bit and isinstance(target, bnb.nn.Linear8bitLt): kwargs.update( { "has_fp16_weights": target.state.has_fp16_weights, @@ -154,7 +154,11 @@ class LoraModel(torch.nn.Module): "index": target.index, } ) - new_module = Linear8bitLt(target.in_features, target.out_features, bias=bias, **kwargs) + if self.peft_config.enable_lora is None: + new_module = Linear8bitLt(target.in_features, target.out_features, bias=bias, **kwargs) + else: + kwargs.update({"enable_lora": self.peft_config.enable_lora}) + new_module = MergedLinear8bitLt(target.in_features, target.out_features, bias=bias, **kwargs) elif isinstance(target, torch.nn.Linear) and self.peft_config.enable_lora is None: new_module = Linear(target.in_features, target.out_features, bias=bias, **kwargs) elif self.peft_config.enable_lora is not None: @@ -509,3 +513,82 @@ if is_bnb_available(): output = self.lora_B(self.lora_A(self.lora_dropout(x))) * self.scaling result += output return result + + class MergedLinear8bitLt(bnb.nn.Linear8bitLt, LoraLayer): + # Lora implemented in a dense layer + def __init__( + self, + in_features: int, + out_features: int, + r: int = 0, + lora_alpha: int = 1, + lora_dropout: float = 0.0, + enable_lora: List[bool] = [False], + **kwargs, + ): + bnb.nn.Linear8bitLt.__init__( + self, + in_features, + out_features, + bias=kwargs.get("bias", True), + has_fp16_weights=kwargs.get("has_fp16_weights", True), + memory_efficient_backward=kwargs.get("memory_efficient_backward", False), + threshold=kwargs.get("threshold", 0.0), + index=kwargs.get("index", None), + ) + LoraLayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, merge_weights=False) + if out_features % len(enable_lora) != 0: + raise ValueError("The length of enable_lora must divide out_features") + self.enable_lora = enable_lora + # Actual trainable parameters + if r > 0 and any(enable_lora): + self.lora_A = nn.Linear(in_features, r * sum(enable_lora), bias=False) + self.lora_B = nn.Conv1d( + r * sum(enable_lora), + out_features // len(enable_lora) * sum(enable_lora), + kernel_size=1, + groups=2, + bias=False, + ) + self.scaling = self.lora_alpha / self.r + # Freezing the pre-trained weight matrix + self.weight.requires_grad = False + # Compute the indices + self.lora_ind = self.weight.new_zeros((out_features,), dtype=torch.bool).view(len(enable_lora), -1) + self.lora_ind[enable_lora, :] = True + self.lora_ind = self.lora_ind.view(-1) + self.reset_parameters() + + def reset_parameters(self): + if hasattr(self, "lora_A"): + # initialize A the same way as the default for nn.Linear and B to zero + nn.init.kaiming_uniform_(self.lora_A.weight, a=math.sqrt(5)) + nn.init.zeros_(self.lora_B.weight) + + def zero_pad(self, x): + result = x.new_zeros((*x.shape[:-1], self.out_features)) + result = result.view(-1, self.out_features) + result[:, self.lora_ind] = x.reshape( + -1, self.out_features // len(self.enable_lora) * sum(self.enable_lora) + ) + return result.view((*x.shape[:-1], self.out_features)) + + def forward(self, x: torch.Tensor): + result = super().forward(x) + if self.disable_adapters: + return result + elif self.r > 0: + if not torch.is_autocast_enabled(): + expected_dtype = result.dtype + if x.dtype != torch.float32: + x = x.float() + after_A = self.lora_A(self.lora_dropout(x)) + after_B = self.lora_B(after_A.transpose(-2, -1)).transpose(-2, -1) + output = self.zero_pad(after_B).to(expected_dtype) * self.scaling + result += output + else: + after_A = self.lora_A(self.lora_dropout(x)) + after_B = self.lora_B(after_A.transpose(-2, -1)).transpose(-2, -1) + output = self.zero_pad(after_B) * self.scaling + result += output + return result diff --git a/src/peft/utils/config.py b/src/peft/utils/config.py index cf1e49a..3e2cf5b 100644 --- a/src/peft/utils/config.py +++ b/src/peft/utils/config.py @@ -160,6 +160,8 @@ class PromptLearningConfig(PeftConfig): token_dim: int = field( default=None, metadata={"help": "The hidden embedding dimension of the base transformer model"} ) - num_transformer_submodules: Optional[int] = field(default=None, metadata={"help": "Number of transformer submodules"}) + num_transformer_submodules: Optional[int] = field( + default=None, metadata={"help": "Number of transformer submodules"} + ) num_attention_heads: Optional[int] = field(default=None, metadata={"help": "Number of attention heads"}) num_layers: Optional[int] = field(default=None, metadata={"help": "Number of transformer layers"}) From a43ef6ec72f322804d6e4ad712705e127ad96ee9 Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Wed, 8 Mar 2023 00:53:08 +0530 Subject: [PATCH 11/28] fixing ds conv1D issue thanks to @dumpmemory --- src/peft/tuners/lora.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/peft/tuners/lora.py b/src/peft/tuners/lora.py index ed10675..301bfd4 100644 --- a/src/peft/tuners/lora.py +++ b/src/peft/tuners/lora.py @@ -164,7 +164,9 @@ class LoraModel(torch.nn.Module): elif self.peft_config.enable_lora is not None: kwargs.update({"enable_lora": self.peft_config.enable_lora}) if isinstance(target, Conv1D): - in_features, out_features = target.weight.shape + in_features, out_features = ( + target.weight.ds_shape if hasattr(target.weight, "ds_shape") else target.weight.shape + ) else: in_features, out_features = target.in_features, target.out_features if kwargs["fan_in_fan_out"]: From 27c2701555db017e247e1dbe687a710af1098883 Mon Sep 17 00:00:00 2001 From: Jason Phang Date: Tue, 7 Mar 2023 19:17:35 -0500 Subject: [PATCH 12/28] Add Prefix Tuning citation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 45fbf93..074549c 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Seamlessly integrated with 🤗 Accelerate for large scale models leveraging Dee Supported methods: 1. LoRA: [LORA: LOW-RANK ADAPTATION OF LARGE LANGUAGE MODELS](https://arxiv.org/pdf/2106.09685.pdf) -2. Prefix Tuning: [P-Tuning v2: Prompt Tuning Can Be Comparable to Fine-tuning Universally Across Scales and Tasks](https://arxiv.org/pdf/2110.07602.pdf) +2. Prefix Tuning: [Prefix-Tuning: Optimizing Continuous Prompts for Generation](https://aclanthology.org/2021.acl-long.353/), [P-Tuning v2: Prompt Tuning Can Be Comparable to Fine-tuning Universally Across Scales and Tasks](https://arxiv.org/pdf/2110.07602.pdf) 3. P-Tuning: [GPT Understands, Too](https://arxiv.org/pdf/2103.10385.pdf) 4. Prompt Tuning: [The Power of Scale for Parameter-Efficient Prompt Tuning](https://arxiv.org/pdf/2104.08691.pdf) From baa2a4d53f4fc337e846808c9feb1253dea081eb Mon Sep 17 00:00:00 2001 From: Jason Phang Date: Tue, 7 Mar 2023 21:05:58 -0500 Subject: [PATCH 13/28] LLaMA support --- README.md | 17 +++++++++-------- src/peft/mapping.py | 1 + 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 45fbf93..686400b 100644 --- a/README.md +++ b/README.md @@ -215,14 +215,15 @@ An example is provided in `~examples/causal_language_modeling/peft_lora_clm_acce ## Models support matrix ### Causal Language Modeling -| Model | LoRA | Prefix Tuning | P-Tuning | Prompt Tuning | -| --------- | ---- | ---- | ---- | ---- | -| GPT-2 | ✅ | ✅ | ✅ | ✅ | -| Bloom | ✅ | ✅ | ✅ | ✅ | -| OPT | ✅ | ✅ | ✅ | ✅ | -| GPT-Neo | ✅ | ✅ | ✅ | ✅ | -| GPT-J | ✅ | ✅ | ✅ | ✅ | -| GPT-NeoX-20B | ✅ | ✅ | ✅ | ✅ | +| Model | LoRA | Prefix Tuning | P-Tuning | Prompt Tuning | +|--------------| ---- | ---- | ---- | ---- | +| GPT-2 | ✅ | ✅ | ✅ | ✅ | +| Bloom | ✅ | ✅ | ✅ | ✅ | +| OPT | ✅ | ✅ | ✅ | ✅ | +| GPT-Neo | ✅ | ✅ | ✅ | ✅ | +| GPT-J | ✅ | ✅ | ✅ | ✅ | +| GPT-NeoX-20B | ✅ | ✅ | ✅ | ✅ | +| LLaMA | ✅ | ✅ | ✅ | ✅ | ### Conditional Generation | Model | LoRA | Prefix Tuning | P-Tuning | Prompt Tuning | diff --git a/src/peft/mapping.py b/src/peft/mapping.py index 68de0c2..383b329 100644 --- a/src/peft/mapping.py +++ b/src/peft/mapping.py @@ -55,6 +55,7 @@ TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING = { "deberta-v2": ["query_proj", "value_proj"], "deberta": ["in_proj"], "layoutlm": ["query", "value"], + "llama": ["q_proj", "v_proj"], } From d2b99c0b6274b37599ddc78a5e45ad2c0b63ca52 Mon Sep 17 00:00:00 2001 From: dumpmemory <64742282+dumpmemory@users.noreply.github.com> Date: Wed, 8 Mar 2023 18:41:30 +0800 Subject: [PATCH 14/28] fix count num_params should be directly used. --- src/peft/peft_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py index b885624..1c0b15b 100644 --- a/src/peft/peft_model.py +++ b/src/peft/peft_model.py @@ -274,7 +274,7 @@ class PeftModel(PushToHubMixin, torch.nn.Module): all_param += num_params if param.requires_grad: - trainable_params += param.numel() + trainable_params += num_params print( f"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}" ) From 48dc4c624e9540d24af41bcde14e66ce744a206b Mon Sep 17 00:00:00 2001 From: alvanli Date: Wed, 8 Mar 2023 09:57:13 -0500 Subject: [PATCH 15/28] Add callback to save to local --- .../peft_bnb_whisper_large_v2_training.ipynb | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/examples/int8_training/peft_bnb_whisper_large_v2_training.ipynb b/examples/int8_training/peft_bnb_whisper_large_v2_training.ipynb index 0f5d5f7..adeeb5e 100644 --- a/examples/int8_training/peft_bnb_whisper_large_v2_training.ipynb +++ b/examples/int8_training/peft_bnb_whisper_large_v2_training.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "id": "5cefac89", "metadata": {}, @@ -10,7 +9,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "090fa3ed", "metadata": {}, @@ -22,7 +20,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "625e47a0", "metadata": {}, @@ -72,7 +69,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "8a528c1a", "metadata": {}, @@ -139,7 +135,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "805b1c56", "metadata": {}, @@ -1205,7 +1200,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "3906d436", "metadata": {}, @@ -1300,6 +1294,13 @@ "source": [ "from transformers import Seq2SeqTrainer\n", "\n", + "# Save model to local\n", + "class PeftSavingCallback(TrainerCallback):\n", + " def on_train_end(self, args, state, control, **kwargs):\n", + " kwargs[\"model\"].save_pretrained(state.best_model_checkpoint)\n", + " pytorch_model_path = os.path.join(state.best_model_checkpoint, \"pytorch_model.bin\")\n", + " os.remove(pytorch_model_path) if os.path.exists(pytorch_model_path) else None\n", + "\n", "trainer = Seq2SeqTrainer(\n", " args=training_args,\n", " model=model,\n", @@ -1308,6 +1309,7 @@ " data_collator=data_collator,\n", " # compute_metrics=compute_metrics,\n", " tokenizer=processor.feature_extractor,\n", + " callbacks=[PeftSavingCallback]\n", ")\n", "model.config.use_cache = False # silence the warnings. Please re-enable for inference!" ] @@ -1585,7 +1587,6 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "id": "Kzfg2qoXgrhg", "metadata": { @@ -1928,7 +1929,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.10.6" }, "widgets": { "application/vnd.jupyter.widget-state+json": { From 5e788b329d27663a056c18449b4c2d4915c04237 Mon Sep 17 00:00:00 2001 From: alvanli Date: Wed, 8 Mar 2023 10:05:53 -0500 Subject: [PATCH 16/28] Use on save callback --- .../peft_bnb_whisper_large_v2_training.ipynb | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/examples/int8_training/peft_bnb_whisper_large_v2_training.ipynb b/examples/int8_training/peft_bnb_whisper_large_v2_training.ipynb index adeeb5e..c8bf47f 100644 --- a/examples/int8_training/peft_bnb_whisper_large_v2_training.ipynb +++ b/examples/int8_training/peft_bnb_whisper_large_v2_training.ipynb @@ -1292,14 +1292,24 @@ } ], "source": [ - "from transformers import Seq2SeqTrainer\n", + "from transformers import Seq2SeqTrainer, TrainerCallback, TrainingArguments, TrainerState, TrainerControl\n", + "from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR\n", "\n", - "# Save model to local\n", - "class PeftSavingCallback(TrainerCallback):\n", - " def on_train_end(self, args, state, control, **kwargs):\n", - " kwargs[\"model\"].save_pretrained(state.best_model_checkpoint)\n", - " pytorch_model_path = os.path.join(state.best_model_checkpoint, \"pytorch_model.bin\")\n", - " os.remove(pytorch_model_path) if os.path.exists(pytorch_model_path) else None\n", + "class SavePeftModelCallback(TrainerCallback):\n", + " def on_save(\n", + " self, args: TrainingArguments, state: TrainerState, control: TrainerControl, **kwargs,\n", + " ):\n", + " checkpoint_folder = os.path.join(\n", + " args.output_dir, f\"{PREFIX_CHECKPOINT_DIR}-{state.global_step}\"\n", + " ) \n", + "\n", + " peft_model_path = os.path.join(checkpoint_folder, \"adapter_model\")\n", + " kwargs[\"model\"].save_pretrained(peft_model_path)\n", + "\n", + " pytorch_model_path = os.path.join(checkpoint_folder, \"pytorch_model.bin\")\n", + " if os.path.exists(pytorch_model_path):\n", + " os.remove(pytorch_model_path)\n", + " return control\n", "\n", "trainer = Seq2SeqTrainer(\n", " args=training_args,\n", @@ -1309,7 +1319,7 @@ " data_collator=data_collator,\n", " # compute_metrics=compute_metrics,\n", " tokenizer=processor.feature_extractor,\n", - " callbacks=[PeftSavingCallback]\n", + " callbacks=[SavePeftModelCallback]\n", ")\n", "model.config.use_cache = False # silence the warnings. Please re-enable for inference!" ] From 4497d6438ccda5f1e22aae248f3411d407593ba5 Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Thu, 9 Mar 2023 08:53:36 +0530 Subject: [PATCH 17/28] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 074549c..90623e4 100644 --- a/README.md +++ b/README.md @@ -126,12 +126,13 @@ Try out the 🤗 Gradio Space which should run seamlessly on a T4 instance: ![peft lora dreambooth gradio space](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/peft_lora_dreambooth_gradio_space.png) ### Parameter Efficient Tuning of LLMs for RLHF components such as Ranker and Policy [ToDo] +Here is an exmaple in trl library on using PEFT+INT8 for tuning policy model: [gpt2-sentiment_peft.py](https://github.com/lvwerra/trl/blob/main/examples/sentiment/scripts/gpt2-sentiment_peft.py) ### INT8 training of large models in Colab using PEFT LoRA and bits_and_bytes Here is now a demo on how to fine tune [OPT-6.7b](https://huggingface.co/facebook/opt-6.7b) (14GB in fp16) in a Google colab: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1jCkpikz0J2o20FBQmYmAGdiKmJGOMo-o?usp=sharing) -Here is now a demo on how to fine tune [whishper-large](openai/whisper-large-v2) (1.5B params) (14GB in fp16) in a Google colab: [ToDo] +Here is now a demo on how to fine tune [whishper-large](openai/whisper-large-v2) (1.5B params) (14GB in fp16) in a Google colab: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1DOkD_5OUjFa0r5Ik3SgywJLJtEo2qLxO?usp=sharing) and [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1vhF8yueFqha3Y3CpTHN6q9EVcII9EYzs?usp=sharing) ### Save compute and storage even for medium and small models From f1980e9be283abc02fe90dd51d36dcd179fc92a7 Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Thu, 9 Mar 2023 08:57:52 +0530 Subject: [PATCH 18/28] minor changes --- README.md | 9 +++++---- .../peft_bnb_whisper_large_v2_training.ipynb | 14 +++++++++----- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 90623e4..93cf3f1 100644 --- a/README.md +++ b/README.md @@ -125,14 +125,15 @@ Try out the 🤗 Gradio Space which should run seamlessly on a T4 instance: ![peft lora dreambooth gradio space](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/peft_lora_dreambooth_gradio_space.png) -### Parameter Efficient Tuning of LLMs for RLHF components such as Ranker and Policy [ToDo] -Here is an exmaple in trl library on using PEFT+INT8 for tuning policy model: [gpt2-sentiment_peft.py](https://github.com/lvwerra/trl/blob/main/examples/sentiment/scripts/gpt2-sentiment_peft.py) +### Parameter Efficient Tuning of LLMs for RLHF components such as Ranker and Policy +- Here is an exmaple in [trl](https://github.com/lvwerra/trl) library using PEFT+INT8 for tuning policy model: [gpt2-sentiment_peft.py](https://github.com/lvwerra/trl/blob/main/examples/sentiment/scripts/gpt2-sentiment_peft.py) +- Example using PEFT for both reward model and policy [ToDo] ### INT8 training of large models in Colab using PEFT LoRA and bits_and_bytes -Here is now a demo on how to fine tune [OPT-6.7b](https://huggingface.co/facebook/opt-6.7b) (14GB in fp16) in a Google colab: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1jCkpikz0J2o20FBQmYmAGdiKmJGOMo-o?usp=sharing) +- Here is now a demo on how to fine tune [OPT-6.7b](https://huggingface.co/facebook/opt-6.7b) (14GB in fp16) in a Google colab: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1jCkpikz0J2o20FBQmYmAGdiKmJGOMo-o?usp=sharing) -Here is now a demo on how to fine tune [whishper-large](openai/whisper-large-v2) (1.5B params) (14GB in fp16) in a Google colab: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1DOkD_5OUjFa0r5Ik3SgywJLJtEo2qLxO?usp=sharing) and [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1vhF8yueFqha3Y3CpTHN6q9EVcII9EYzs?usp=sharing) +- Here is now a demo on how to fine tune [whishper-large](openai/whisper-large-v2) (1.5B params) (14GB in fp16) in a Google colab: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1DOkD_5OUjFa0r5Ik3SgywJLJtEo2qLxO?usp=sharing) and [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1vhF8yueFqha3Y3CpTHN6q9EVcII9EYzs?usp=sharing) ### Save compute and storage even for medium and small models diff --git a/examples/int8_training/peft_bnb_whisper_large_v2_training.ipynb b/examples/int8_training/peft_bnb_whisper_large_v2_training.ipynb index c8bf47f..3be7f9b 100644 --- a/examples/int8_training/peft_bnb_whisper_large_v2_training.ipynb +++ b/examples/int8_training/peft_bnb_whisper_large_v2_training.ipynb @@ -1295,13 +1295,16 @@ "from transformers import Seq2SeqTrainer, TrainerCallback, TrainingArguments, TrainerState, TrainerControl\n", "from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR\n", "\n", + "\n", "class SavePeftModelCallback(TrainerCallback):\n", " def on_save(\n", - " self, args: TrainingArguments, state: TrainerState, control: TrainerControl, **kwargs,\n", + " self,\n", + " args: TrainingArguments,\n", + " state: TrainerState,\n", + " control: TrainerControl,\n", + " **kwargs,\n", " ):\n", - " checkpoint_folder = os.path.join(\n", - " args.output_dir, f\"{PREFIX_CHECKPOINT_DIR}-{state.global_step}\"\n", - " ) \n", + " checkpoint_folder = os.path.join(args.output_dir, f\"{PREFIX_CHECKPOINT_DIR}-{state.global_step}\")\n", "\n", " peft_model_path = os.path.join(checkpoint_folder, \"adapter_model\")\n", " kwargs[\"model\"].save_pretrained(peft_model_path)\n", @@ -1311,6 +1314,7 @@ " os.remove(pytorch_model_path)\n", " return control\n", "\n", + "\n", "trainer = Seq2SeqTrainer(\n", " args=training_args,\n", " model=model,\n", @@ -1319,7 +1323,7 @@ " data_collator=data_collator,\n", " # compute_metrics=compute_metrics,\n", " tokenizer=processor.feature_extractor,\n", - " callbacks=[SavePeftModelCallback]\n", + " callbacks=[SavePeftModelCallback],\n", ")\n", "model.config.use_cache = False # silence the warnings. Please re-enable for inference!" ] From 80c96de2776e64ece7e1637fcda7a3d6a798b674 Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Thu, 9 Mar 2023 09:23:24 +0530 Subject: [PATCH 19/28] release v0.3.0.dev0 --- setup.py | 2 +- src/peft/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 6b26477..8d15e01 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ extras["dev"] = extras["quality"] + extras["docs_specific"] setup( name="peft", - version="0.2.0.dev0", + version="0.3.0.dev0", description="Parameter-Efficient Fine-Tuning (PEFT)", license_files=["LICENSE"], long_description=open("README.md", "r", encoding="utf-8").read(), diff --git a/src/peft/__init__.py b/src/peft/__init__.py index 3dd7acf..e141347 100644 --- a/src/peft/__init__.py +++ b/src/peft/__init__.py @@ -17,7 +17,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.2.0.dev0" +__version__ = "0.3.0.dev0" from .mapping import MODEL_TYPE_TO_PEFT_MODEL_MAPPING, PEFT_TYPE_TO_CONFIG_MAPPING, get_peft_config, get_peft_model from .peft_model import ( From e85c18f019760b9a46f98d30e5e6132ba077d8d4 Mon Sep 17 00:00:00 2001 From: dumpmemory <64742282+dumpmemory@users.noreply.github.com> Date: Thu, 9 Mar 2023 22:01:16 +0800 Subject: [PATCH 20/28] Update README.md add one caveat situation for using LoRA + ZeRO 3 setting. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 93cf3f1..4ceabee 100644 --- a/README.md +++ b/README.md @@ -344,6 +344,7 @@ any GPU memory savings. Please refer issue [[FSDP] FSDP with CPU offload consume `P_TUNING`/`PROMPT_TUNING` appends soft prompt embeddings to `input_embeds` to create new `input_embeds` to be given to the model. Therefore, `generate` doesn't support this yet. +4. For causal_language_modeling with LoRA, like GPT2 models, we might need to set zero3_init_flag=false in accelerate config.yaml. The related issue is [[BUG] memory leak under zero.Init](https://github.com/microsoft/DeepSpeed/issues/2637) ## Backlog: 1. Explore and possibly integrate `(IA)^3` 2. Add tests From 354bea87194d2f224b6d2d26a6278b837fa39f0c Mon Sep 17 00:00:00 2001 From: dumpmemory <64742282+dumpmemory@users.noreply.github.com> Date: Thu, 9 Mar 2023 22:03:40 +0800 Subject: [PATCH 21/28] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4ceabee..4a31d82 100644 --- a/README.md +++ b/README.md @@ -344,7 +344,7 @@ any GPU memory savings. Please refer issue [[FSDP] FSDP with CPU offload consume `P_TUNING`/`PROMPT_TUNING` appends soft prompt embeddings to `input_embeds` to create new `input_embeds` to be given to the model. Therefore, `generate` doesn't support this yet. -4. For causal_language_modeling with LoRA, like GPT2 models, we might need to set zero3_init_flag=false in accelerate config.yaml. The related issue is [[BUG] memory leak under zero.Init](https://github.com/microsoft/DeepSpeed/issues/2637) +4. When using ZeRO3 with zero3_init_flag=True, if you find the gpu memory increase with training steps. we might need to set zero3_init_flag=false in accelerate config.yaml. The related issue is [[BUG] memory leak under zero.Init](https://github.com/microsoft/DeepSpeed/issues/2637) ## Backlog: 1. Explore and possibly integrate `(IA)^3` 2. Add tests From 644d68ee6f6b3b0b769344e6c7bc9dfae8ced02b Mon Sep 17 00:00:00 2001 From: PanQiWei <594557445@qq.com> Date: Fri, 10 Mar 2023 10:36:11 +0800 Subject: [PATCH 22/28] changed: 1. replace base_model.prepare_inputs_for_generation and base_model._prepare_encoder_decoder_kwargs_for_generation temporarily --- src/peft/peft_model.py | 110 +++++++++++++++++++++++++---------------- 1 file changed, 67 insertions(+), 43 deletions(-) diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py index cdb2b39..f73a66a 100644 --- a/src/peft/peft_model.py +++ b/src/peft/peft_model.py @@ -513,7 +513,6 @@ class PeftModelForCausalLM(PeftModel): def __init__(self, model, peft_config: PeftConfig): super().__init__(model, peft_config) self.base_model_prepare_inputs_for_generation = self.base_model.prepare_inputs_for_generation - self.base_model.prepare_inputs_for_generation = self.prepare_inputs_for_generation def forward( self, @@ -576,28 +575,38 @@ class PeftModelForCausalLM(PeftModel): return self.base_model(inputs_embeds=inputs_embeds, **kwargs) def generate(self, **kwargs): - if not isinstance(self.peft_config, PromptLearningConfig): - return self.base_model.generate(**kwargs) + self.base_model.prepare_inputs_for_generation = self.prepare_inputs_for_generation + try: + if not isinstance(self.peft_config, PromptLearningConfig): + outputs = self.base_model.generate(**kwargs) + else: + if "input_ids" not in kwargs: + raise ValueError("input_ids must be provided for Peft model generation") + if kwargs.get("attention_mask", None) is not None: + # concat prompt attention mask + prefix_attention_mask = torch.ones( + kwargs["input_ids"].shape[0], self.peft_config.num_virtual_tokens + ).to(kwargs["input_ids"].device) + kwargs["attention_mask"] = torch.cat((prefix_attention_mask, kwargs["attention_mask"]), dim=1) + + if kwargs.get("position_ids", None) is not None: + warnings.warn( + "Position ids are not supported for parameter efficient tuning. Ignoring position ids." + ) + kwargs["position_ids"] = None + if kwargs.get("token_type_ids", None) is not None: + warnings.warn( + "Token type ids are not supported for parameter efficient tuning. Ignoring token type ids" + ) + kwargs["token_type_ids"] = None + + outputs = self.base_model.generate(**kwargs) + except: + self.base_model.prepare_inputs_for_generation = self.base_model_prepare_inputs_for_generation + raise else: - if "input_ids" not in kwargs: - raise ValueError("input_ids must be provided for Peft model generation") - if kwargs.get("attention_mask", None) is not None: - # concat prompt attention mask - prefix_attention_mask = torch.ones( - kwargs["input_ids"].shape[0], self.peft_config.num_virtual_tokens - ).to(kwargs["input_ids"].device) - kwargs["attention_mask"] = torch.cat((prefix_attention_mask, kwargs["attention_mask"]), dim=1) - - if kwargs.get("position_ids", None) is not None: - warnings.warn("Position ids are not supported for parameter efficient tuning. Ignoring position ids.") - kwargs["position_ids"] = None - if kwargs.get("token_type_ids", None) is not None: - warnings.warn( - "Token type ids are not supported for parameter efficient tuning. Ignoring token type ids" - ) - kwargs["token_type_ids"] = None - - return self.base_model.generate(**kwargs) + self.base_model.prepare_inputs_for_generation = self.base_model_prepare_inputs_for_generation + return outputs def prepare_inputs_for_generation(self, *args, **kwargs): model_kwargs = self.base_model_prepare_inputs_for_generation(*args, **kwargs) @@ -641,13 +650,9 @@ class PeftModelForSeq2SeqLM(PeftModel): def __init__(self, model, peft_config: PeftConfig): super().__init__(model, peft_config) self.base_model_prepare_inputs_for_generation = self.base_model.prepare_inputs_for_generation - self.base_model.prepare_inputs_for_generation = self.prepare_inputs_for_generation self.base_model_prepare_encoder_decoder_kwargs_for_generation = ( self.base_model._prepare_encoder_decoder_kwargs_for_generation ) - self.base_model._prepare_encoder_decoder_kwargs_for_generation = ( - self._prepare_encoder_decoder_kwargs_for_generation - ) def forward( self, @@ -740,24 +745,43 @@ class PeftModelForSeq2SeqLM(PeftModel): ) def generate(self, **kwargs): - if not isinstance(self.peft_config, PromptLearningConfig): - return self.base_model.generate(**kwargs) - else: - if "input_ids" not in kwargs: - raise ValueError("input_ids must be provided for Peft model generation") - if kwargs.get("position_ids", None) is not None: - warnings.warn("Position ids are not supported for parameter efficient tuning. Ignoring position ids.") - kwargs["position_ids"] = None - if kwargs.get("token_type_ids", None) is not None: - warnings.warn( - "Token type ids are not supported for parameter efficient tuning. Ignoring token type ids" - ) - kwargs["token_type_ids"] = None - - if self.peft_config.peft_type == PeftType.PREFIX_TUNING: - return self.base_model.generate(**kwargs) + self.base_model.prepare_inputs_for_generation = self.prepare_inputs_for_generation + self.base_model._prepare_encoder_decoder_kwargs_for_generation = ( + self._prepare_encoder_decoder_kwargs_for_generation + ) + try: + if not isinstance(self.peft_config, PromptLearningConfig): + outputs = self.base_model.generate(**kwargs) else: - raise NotImplementedError + if "input_ids" not in kwargs: + raise ValueError("input_ids must be provided for Peft model generation") + if kwargs.get("position_ids", None) is not None: + warnings.warn( + "Position ids are not supported for parameter efficient tuning. Ignoring position ids." + ) + kwargs["position_ids"] = None + if kwargs.get("token_type_ids", None) is not None: + warnings.warn( + "Token type ids are not supported for parameter efficient tuning. Ignoring token type ids" + ) + kwargs["token_type_ids"] = None + + if self.peft_config.peft_type == PeftType.PREFIX_TUNING: + outputs = self.base_model.generate(**kwargs) + else: + raise NotImplementedError + except: + self.base_model.prepare_inputs_for_generation = self.base_model_prepare_inputs_for_generation + self.base_model._prepare_encoder_decoder_kwargs_for_generation = ( + self.base_model_prepare_encoder_decoder_kwargs_for_generation + ) + raise + else: + self.base_model.prepare_inputs_for_generation = self.base_model_prepare_inputs_for_generation + self.base_model._prepare_encoder_decoder_kwargs_for_generation = ( + self.base_model_prepare_encoder_decoder_kwargs_for_generation + ) + return outputs def prepare_inputs_for_generation(self, *args, **kwargs): model_kwargs = self.base_model_prepare_inputs_for_generation(*args, **kwargs) From 43cb7040c63fe7c94e5793a03444ddfe43d9f3a8 Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Mon, 13 Mar 2023 16:01:28 +0530 Subject: [PATCH 23/28] fixing merged_linear lora issues --- src/peft/tuners/lora.py | 54 ++++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/src/peft/tuners/lora.py b/src/peft/tuners/lora.py index 301bfd4..22d1a1e 100644 --- a/src/peft/tuners/lora.py +++ b/src/peft/tuners/lora.py @@ -127,12 +127,14 @@ class LoraModel(torch.nn.Module): "You can install it with `pip install bitsandbytes`." ) is_target_modules_in_base_model = False + is_hf_device_map_available = hasattr(self.model, "hf_device_map") kwargs = { "r": self.peft_config.r, "lora_alpha": self.peft_config.lora_alpha, "lora_dropout": self.peft_config.lora_dropout, "fan_in_fan_out": self.peft_config.fan_in_fan_out, - "merge_weights": self.peft_config.merge_weights or self.peft_config.inference_mode, + "merge_weights": (self.peft_config.merge_weights or self.peft_config.inference_mode) + and not is_hf_device_map_available, } key_list = [key for key, _ in self.model.named_modules()] for key in key_list: @@ -174,7 +176,7 @@ class LoraModel(torch.nn.Module): "fan_in_fan_out is set to True but the target module is not a Conv1D. " "Setting fan_in_fan_out to False." ) - kwargs["fan_in_fan_out"] = False + kwargs["fan_in_fan_out"] = self.peft_config.fan_in_fan_out = False new_module = MergedLinear(in_features, out_features, bias=bias, **kwargs) self._replace_module(parent, target_name, new_module, target) if not is_target_modules_in_base_model: @@ -414,22 +416,30 @@ class MergedLinear(nn.Linear, LoraLayer): if not mode and self.merge_weights and not self.merged: # Merge the weights and mark it if self.r > 0 and any(self.enable_lora): - delta_w = F.conv1d( - self.lora_A.weight.data.unsqueeze(0), - self.lora_B.weight.data.unsqueeze(-1), - groups=sum(self.enable_lora), - ).squeeze(0) - self.weight.data += self.zero_pad(transpose(delta_w * self.scaling, self.fan_in_fan_out)) + delta_w = ( + F.conv1d( + self.lora_A.weight.data.unsqueeze(0), + self.lora_B.weight.data, + groups=sum(self.enable_lora), + ) + .squeeze(0) + .transpose(-2, -1) + ) + self.weight.data += transpose(self.zero_pad(delta_w * self.scaling), not self.fan_in_fan_out) self.merged = True elif self.merge_weights and self.merged: # Make sure that the weights are not merged if self.r > 0 and any(self.enable_lora): - delta_w = F.conv1d( - self.lora_A.weight.data.unsqueeze(0), - self.lora_B.weight.data.unsqueeze(-1), - groups=sum(self.enable_lora), - ).squeeze(0) - self.weight.data -= self.zero_pad(transpose(delta_w * self.scaling, self.fan_in_fan_out)) + delta_w = ( + F.conv1d( + self.lora_A.weight.data.unsqueeze(0), + self.lora_B.weight.data, + groups=sum(self.enable_lora), + ) + .squeeze(0) + .transpose(-2, -1) + ) + self.weight.data -= transpose(self.zero_pad(delta_w * self.scaling), not self.fan_in_fan_out) self.merged = False def eval(self): @@ -440,12 +450,16 @@ class MergedLinear(nn.Linear, LoraLayer): def forward(self, x: torch.Tensor): if self.disable_adapters: if self.r > 0 and self.merged and any(self.enable_lora): - delta_w = F.conv1d( - self.lora_A.weight.data.unsqueeze(0), - self.lora_B.weight.data.unsqueeze(-1), - groups=sum(self.enable_lora), - ).squeeze(0) - self.weight.data -= self.zero_pad(transpose(delta_w * self.scaling, self.fan_in_fan_out)) + delta_w = ( + F.conv1d( + self.lora_A.weight.data.unsqueeze(0), + self.lora_B.weight.data, + groups=sum(self.enable_lora), + ) + .squeeze(0) + .transpose(-2, -1) + ) + self.weight.data -= transpose(self.zero_pad(delta_w * self.scaling), not self.fan_in_fan_out) self.merged = False return F.linear(x, transpose(self.weight, self.fan_in_fan_out), bias=self.bias) elif self.merged: From df0e1fb59266c9903ddd6dbfe7339bcd2068d150 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 14 Mar 2023 13:06:33 +0100 Subject: [PATCH 24/28] [`core`] Fix peft multi-gpu issue (#145) * add multi-gpu support * rm deepcopy * tryo to comment * style * fix nits --- src/peft/tuners/lora.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/peft/tuners/lora.py b/src/peft/tuners/lora.py index 22d1a1e..0f65cbf 100644 --- a/src/peft/tuners/lora.py +++ b/src/peft/tuners/lora.py @@ -200,6 +200,11 @@ class LoraModel(torch.nn.Module): new_module.state = old_module.state new_module.to(old_module.weight.device) + # dispatch to correct device + for name, module in new_module.named_modules(): + if "lora_" in name: + module.to(old_module.weight.device) + def __getattr__(self, name: str): """Forward missing attributes to the wrapped module.""" try: @@ -345,6 +350,7 @@ class Linear(nn.Linear, LoraLayer): transpose(self.lora_B.weight @ self.lora_A.weight, self.fan_in_fan_out) * self.scaling ) self.merged = False + return F.linear(x, transpose(self.weight, self.fan_in_fan_out), bias=self.bias) elif self.r > 0 and not self.merged: result = F.linear(x, transpose(self.weight, self.fan_in_fan_out), bias=self.bias) From 54b6ce2c0e19eed7fbba1f101c278adde64952ab Mon Sep 17 00:00:00 2001 From: mymusise Date: Thu, 16 Mar 2023 19:09:12 +0800 Subject: [PATCH 25/28] ChatGLM support Signed-off-by: mymusise --- README.md | 1 + src/peft/mapping.py | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index 49bfcd9..3a9726e 100644 --- a/README.md +++ b/README.md @@ -226,6 +226,7 @@ An example is provided in `~examples/causal_language_modeling/peft_lora_clm_acce | GPT-J | ✅ | ✅ | ✅ | ✅ | | GPT-NeoX-20B | ✅ | ✅ | ✅ | ✅ | | LLaMA | ✅ | ✅ | ✅ | ✅ | +| ChatGLM | ✅ | ✅ | ✅ | ✅ | ### Conditional Generation | Model | LoRA | Prefix Tuning | P-Tuning | Prompt Tuning | diff --git a/src/peft/mapping.py b/src/peft/mapping.py index 383b329..dbb9f36 100644 --- a/src/peft/mapping.py +++ b/src/peft/mapping.py @@ -56,6 +56,7 @@ TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING = { "deberta": ["in_proj"], "layoutlm": ["query", "value"], "llama": ["q_proj", "v_proj"], + "chatglm": ["query_key_value"], } From b5b3ae3cbe23243af6fe4815a481e925e634112a Mon Sep 17 00:00:00 2001 From: Haofan Wang Date: Tue, 21 Mar 2023 21:38:15 +0800 Subject: [PATCH 26/28] Update train_dreambooth.py --- examples/lora_dreambooth/train_dreambooth.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/examples/lora_dreambooth/train_dreambooth.py b/examples/lora_dreambooth/train_dreambooth.py index c06a175..9145eca 100644 --- a/examples/lora_dreambooth/train_dreambooth.py +++ b/examples/lora_dreambooth/train_dreambooth.py @@ -999,7 +999,10 @@ def main(args): pipeline.set_progress_bar_config(disable=True) # run inference - generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + if args.seed is not None: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + else: + generator = None images = [] for _ in range(args.num_validation_images): image = pipeline(args.validation_prompt, num_inference_steps=25, generator=generator).images[0] @@ -1050,15 +1053,17 @@ def main(args): if accelerator.is_main_process: if args.use_lora: lora_config = {} - state_dict = get_peft_model_state_dict(unet, state_dict=accelerator.get_state_dict(unet)) - lora_config["peft_config"] = unet.get_peft_config_as_dict(inference=True) + unwarpped_unet = accelerator.unwrap_model(unet) + state_dict = get_peft_model_state_dict(unwarpped_unet, state_dict=accelerator.get_state_dict(unet)) + lora_config["peft_config"] = unwarpped_unet.get_peft_config_as_dict(inference=True) if args.train_text_encoder: + unwarpped_text_encoder = accelerator.unwrap_model(text_encoder) text_encoder_state_dict = get_peft_model_state_dict( - text_encoder, state_dict=accelerator.get_state_dict(text_encoder) + unwarpped_text_encoder, state_dict=accelerator.get_state_dict(text_encoder) ) text_encoder_state_dict = {f"text_encoder_{k}": v for k, v in text_encoder_state_dict.items()} state_dict.update(text_encoder_state_dict) - lora_config["text_encoder_peft_config"] = text_encoder.get_peft_config_as_dict(inference=True) + lora_config["text_encoder_peft_config"] = unwarpped_text_encoder.get_peft_config_as_dict(inference=True) accelerator.print(state_dict) accelerator.save(state_dict, os.path.join(args.output_dir, f"{args.instance_prompt}_lora.pt")) From 2632e7eba7fd06da522483caa1e676922258aaa9 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Thu, 23 Mar 2023 12:38:40 +0100 Subject: [PATCH 27/28] [`CI`] Add ci tests (#203) * add ci tests * fix some tests * fix tests * rename * fix * update tests * try * temp hotfix * refactor tests * Update .github/workflows/tests.yml * fix test --- .github/workflows/tests.yml | 46 +++++++ Makefile | 4 +- tests/__init__.py | 0 tests/test_peft_model.py | 243 +++++++++++++++++------------------- tests/testing_common.py | 103 +++++++++++++++ tests/testing_utils.py | 49 ++++++++ 6 files changed, 314 insertions(+), 131 deletions(-) create mode 100644 .github/workflows/tests.yml create mode 100644 tests/__init__.py create mode 100644 tests/testing_common.py create mode 100644 tests/testing_utils.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..8bb2491 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,46 @@ +name: tests + +on: + push: + branches: [ main ] + pull_request: + +jobs: + + check_code_quality: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.8" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install .[dev] + - name: Check quality + run: | + make quality + + tests: + needs: check_code_quality + strategy: + matrix: + python-version: [3.8, 3.9, 3.10] + os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + # cpu version of pytorch + pip install .[test] + - name: Test with pytest + run: | + make test \ No newline at end of file diff --git a/Makefile b/Makefile index ff8ed42..61549db 100644 --- a/Makefile +++ b/Makefile @@ -15,4 +15,6 @@ style: black $(check_dirs) ruff $(check_dirs) --fix doc-builder style src tests --max_len 119 - \ No newline at end of file + +test: + pytest tests/ \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_peft_model.py b/tests/test_peft_model.py index 0e85c7c..2ca4895 100644 --- a/tests/test_peft_model.py +++ b/tests/test_peft_model.py @@ -17,157 +17,140 @@ import tempfile import unittest import torch +from parameterized import parameterized from transformers import AutoModelForCausalLM from peft import ( - LoraConfig, PeftModel, - PrefixTuningConfig, - PromptEncoderConfig, - PromptTuningConfig, get_peft_model, get_peft_model_state_dict, - prepare_model_for_training, + prepare_model_for_int8_training, ) +from .testing_common import PeftTestConfigManager + + +# This has to be in the order: model_id, lora_kwargs, prefix_tuning_kwargs, prompt_encoder_kwargs, prompt_tuning_kwargs +PEFT_MODELS_TO_TEST = [ + ("hf-internal-testing/tiny-random-OPTForCausalLM", {"target_modules": ["q_proj", "v_proj"]}, {}, {}, {}), +] + class PeftTestMixin: - checkpoints_to_test = [ - "hf-internal-testing/tiny-random-OPTForCausalLM", - ] - config_classes = ( - LoraConfig, - PrefixTuningConfig, - PromptEncoderConfig, - PromptTuningConfig, - ) - config_kwargs = ( - { - "r": 8, - "lora_alpha": 32, - "target_modules": ["q_proj", "v_proj"], - "lora_dropout": 0.05, - "bias": "none", - "task_type": "CAUSAL_LM", - }, - { - "num_virtual_tokens": 10, - "task_type": "CAUSAL_LM", - }, - { - "num_virtual_tokens": 10, - "encoder_hidden_size": 32, - "task_type": "CAUSAL_LM", - }, - { - "num_virtual_tokens": 10, - "task_type": "CAUSAL_LM", - }, - ) + torch_device = "cuda" if torch.cuda.is_available() else "cpu" class PeftModelTester(unittest.TestCase, PeftTestMixin): r""" Test if the PeftModel behaves as expected. This includes: - test if the model has the expected methods + + We use parametrized.expand for debugging purposes to test each model individually. """ - def test_attributes_model(self): - for model_id in self.checkpoints_to_test: - for i, config_cls in enumerate(self.config_classes): - model = AutoModelForCausalLM.from_pretrained(model_id) - config = config_cls( - base_model_name_or_path=model_id, - **self.config_kwargs[i], + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_MODELS_TO_TEST)) + def test_attributes_parametrized(self, test_name, model_id, config_cls, config_kwargs): + self._test_model_attr(model_id, config_cls, config_kwargs) + + def _test_model_attr(self, model_id, config_cls, config_kwargs): + model = AutoModelForCausalLM.from_pretrained(model_id) + config = config_cls( + base_model_name_or_path=model_id, + **config_kwargs, + ) + model = get_peft_model(model, config) + + self.assertTrue(hasattr(model, "save_pretrained")) + self.assertTrue(hasattr(model, "from_pretrained")) + self.assertTrue(hasattr(model, "push_to_hub")) + + def _test_prepare_for_training(self, model_id, config_cls, config_kwargs): + model = AutoModelForCausalLM.from_pretrained(model_id).to(self.torch_device) + config = config_cls( + base_model_name_or_path=model_id, + **config_kwargs, + ) + model = get_peft_model(model, config) + + dummy_input = torch.LongTensor([[1, 1, 1]]).to(self.torch_device) + dummy_output = model.get_input_embeddings()(dummy_input) + + self.assertTrue(not dummy_output.requires_grad) + + # load with `prepare_model_for_int8_training` + model = AutoModelForCausalLM.from_pretrained(model_id).to(self.torch_device) + model = prepare_model_for_int8_training(model) + + for param in model.parameters(): + self.assertTrue(not param.requires_grad) + + config = config_cls( + base_model_name_or_path=model_id, + **config_kwargs, + ) + model = get_peft_model(model, config) + + # For backward compatibility + if hasattr(model, "enable_input_require_grads"): + model.enable_input_require_grads() + else: + + def make_inputs_require_grad(module, input, output): + output.requires_grad_(True) + + model.get_input_embeddings().register_forward_hook(make_inputs_require_grad) + + dummy_input = torch.LongTensor([[1, 1, 1]]).to(self.torch_device) + dummy_output = model.get_input_embeddings()(dummy_input) + + self.assertTrue(dummy_output.requires_grad) + + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_MODELS_TO_TEST)) + def test_prepare_for_training_parametrized(self, test_name, model_id, config_cls, config_kwargs): + self._test_prepare_for_training(model_id, config_cls, config_kwargs) + + def _test_save_pretrained(self, model_id, config_cls, config_kwargs): + model = AutoModelForCausalLM.from_pretrained(model_id) + config = config_cls( + base_model_name_or_path=model_id, + **config_kwargs, + ) + model = get_peft_model(model, config) + model = model.to(self.torch_device) + + with tempfile.TemporaryDirectory() as tmp_dirname: + model.save_pretrained(tmp_dirname) + + model_from_pretrained = AutoModelForCausalLM.from_pretrained(model_id) + model_from_pretrained = PeftModel.from_pretrained(model_from_pretrained, tmp_dirname) + + # check if the state dicts are equal + state_dict = get_peft_model_state_dict(model) + state_dict_from_pretrained = get_peft_model_state_dict(model_from_pretrained) + + # check if same keys + self.assertEqual(state_dict.keys(), state_dict_from_pretrained.keys()) + + # check if tensors equal + for key in state_dict.keys(): + self.assertTrue( + torch.allclose( + state_dict[key].to(self.torch_device), state_dict_from_pretrained[key].to(self.torch_device) + ) ) - model = get_peft_model(model, config) - self.assertTrue(hasattr(model, "save_pretrained")) - self.assertTrue(hasattr(model, "from_pretrained")) - self.assertTrue(hasattr(model, "push_to_hub")) + # check if `adapter_model.bin` is present + self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.bin"))) - def test_prepare_for_training(self): - r""" - A test that checks if `prepare_for_training` behaves as expected - """ - for model_id in self.checkpoints_to_test: - for i, config_cls in enumerate(self.config_classes): - model = AutoModelForCausalLM.from_pretrained(model_id) - config = config_cls( - base_model_name_or_path=model_id, - **self.config_kwargs[i], - ) - model = get_peft_model(model, config) + # check if `adapter_config.json` is present + self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json"))) - dummy_input = torch.LongTensor([[1, 1, 1]]) - dummy_output = model.get_input_embeddings()(dummy_input) + # check if `pytorch_model.bin` is not present + self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "pytorch_model.bin"))) - self.assertTrue(not dummy_output.requires_grad) + # check if `config.json` is not present + self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json"))) - # load with `prepare_model_for_training` - model = AutoModelForCausalLM.from_pretrained(model_id) - model = prepare_model_for_training(model) - - for param in model.parameters(): - self.assertTrue(not param.requires_grad) - - config = config_cls( - base_model_name_or_path=model_id, - **self.config_kwargs[i], - ) - model = get_peft_model(model, config) - - dummy_input = torch.LongTensor([[1, 1, 1]]) - dummy_output = model.get_input_embeddings()(dummy_input) - - self.assertTrue(dummy_output.requires_grad) - - def test_save_pretrained(self): - r""" - A test to check if `save_pretrained` behaves as expected. This function should only save the state dict of the - adapter model and not the state dict of the base model. Hence inside each saved directory you should have: - - - README.md (that contains an entry `base_model`) - - adapter_config.json - - adapter_model.bin - - """ - for model_id in self.checkpoints_to_test: - for i, config_cls in enumerate(self.config_classes): - model = AutoModelForCausalLM.from_pretrained(model_id) - config = config_cls( - base_model_name_or_path=model_id, - **self.config_kwargs[i], - ) - model = get_peft_model(model, config) - model.to(model.device) - - with tempfile.TemporaryDirectory() as tmp_dirname: - model.save_pretrained(tmp_dirname) - - model_from_pretrained = AutoModelForCausalLM.from_pretrained(model_id) - model_from_pretrained = PeftModel.from_pretrained(model_from_pretrained, tmp_dirname) - model_from_pretrained.to(model.device) - - # check if the state dicts are equal - state_dict = get_peft_model_state_dict(model) - state_dict_from_pretrained = get_peft_model_state_dict(model_from_pretrained) - - # check if same keys - self.assertEqual(state_dict.keys(), state_dict_from_pretrained.keys()) - - # check if tensors equal - for key in state_dict.keys(): - self.assertTrue(torch.allclose(state_dict[key], state_dict_from_pretrained[key])) - - # check if `adapter_model.bin` is present - self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_model.bin"))) - - # check if `adapter_config.json` is present - self.assertTrue(os.path.exists(os.path.join(tmp_dirname, "adapter_config.json"))) - - # check if `pytorch_model.bin` is not present - self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "pytorch_model.bin"))) - - # check if `config.json` is not present - self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json"))) + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_MODELS_TO_TEST)) + def test_save_pretrained(self, test_name, model_id, config_cls, config_kwargs): + self._test_save_pretrained(model_id, config_cls, config_kwargs) diff --git a/tests/testing_common.py b/tests/testing_common.py new file mode 100644 index 0000000..dfdf1d8 --- /dev/null +++ b/tests/testing_common.py @@ -0,0 +1,103 @@ +# coding=utf-8 +# Copyright 2023-present the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections import OrderedDict + +from peft import ( + LoraConfig, + PrefixTuningConfig, + PromptEncoderConfig, + PromptTuningConfig, +) + + +CONFIG_CLASSES = ( + LoraConfig, + PrefixTuningConfig, + PromptEncoderConfig, + PromptTuningConfig, +) +CONFIG_TESTING_KWARGS = ( + { + "r": 8, + "lora_alpha": 32, + "target_modules": None, + "lora_dropout": 0.05, + "bias": "none", + "task_type": "CAUSAL_LM", + }, + { + "num_virtual_tokens": 10, + "task_type": "CAUSAL_LM", + }, + { + "num_virtual_tokens": 10, + "encoder_hidden_size": 32, + "task_type": "CAUSAL_LM", + }, + { + "num_virtual_tokens": 10, + "task_type": "CAUSAL_LM", + }, +) + +CLASSES_MAPPING = { + "lora": (LoraConfig, CONFIG_TESTING_KWARGS[0]), + "prefix_tuning": (PrefixTuningConfig, CONFIG_TESTING_KWARGS[1]), + "prompt_encoder": (PromptEncoderConfig, CONFIG_TESTING_KWARGS[2]), + "prompt_tuning": (PromptTuningConfig, CONFIG_TESTING_KWARGS[3]), +} + + +# Adapted from https://github.com/huggingface/transformers/blob/48327c57182fdade7f7797d1eaad2d166de5c55b/src/transformers/activations.py#LL166C7-L166C22 +class ClassInstantier(OrderedDict): + def __getitem__(self, key, *args, **kwargs): + # check if any of the kwargs is inside the config class kwargs + if any([kwarg in self[key][1] for kwarg in kwargs]): + new_config_kwargs = self[key][1].copy() + new_config_kwargs.update(kwargs) + return (self[key][0], new_config_kwargs) + + return super().__getitem__(key, *args, **kwargs) + + def get_grid_parameters(self, model_list): + r""" + Returns a list of all possible combinations of the parameters in the config classes. + """ + grid_parameters = [] + for model_tuple in model_list: + model_id, lora_kwargs, prefix_tuning_kwargs, prompt_encoder_kwargs, prompt_tuning_kwargs = model_tuple + for key, value in self.items(): + if key == "lora": + # update value[1] if necessary + if lora_kwargs is not None: + value[1].update(lora_kwargs) + elif key == "prefix_tuning": + # update value[1] if necessary + if prefix_tuning_kwargs is not None: + value[1].update(prefix_tuning_kwargs) + elif key == "prompt_encoder": + # update value[1] if necessary + if prompt_encoder_kwargs is not None: + value[1].update(prompt_encoder_kwargs) + else: + # update value[1] if necessary + if prompt_tuning_kwargs is not None: + value[1].update(prompt_tuning_kwargs) + grid_parameters.append((f"test_{model_id}_{key}", model_id, value[0], value[1])) + + return grid_parameters + + +PeftTestConfigManager = ClassInstantier(CLASSES_MAPPING) diff --git a/tests/testing_utils.py b/tests/testing_utils.py new file mode 100644 index 0000000..68851ff --- /dev/null +++ b/tests/testing_utils.py @@ -0,0 +1,49 @@ +# coding=utf-8 +# Copyright 2023-present the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +import torch + + +def require_torch_gpu(test_case): + """ + Decorator marking a test that requires a GPU. Will be skipped when no GPU is available. + """ + if not torch.cuda.is_available(): + return unittest.skip("test requires GPU")(test_case) + else: + return test_case + + +def require_torch_multi_gpu(test_case): + """ + Decorator marking a test that requires multiple GPUs. Will be skipped when less than 2 GPUs are available. + """ + if not torch.cuda.is_available() or torch.cuda.device_count() < 2: + return unittest.skip("test requires multiple GPUs")(test_case) + else: + return test_case + + +def require_bitsandbytes(test_case): + """ + Decorator marking a test that requires the bitsandbytes library. Will be skipped when the library is not installed. + """ + try: + import bitsandbytes # noqa: F401 + except ImportError: + return unittest.skip("test requires bitsandbytes")(test_case) + else: + return test_case From d8c3b6bca49e4aa6e0498b416ed9adc50cc1a5fd Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Thu, 23 Mar 2023 12:54:52 +0100 Subject: [PATCH 28/28] Fix CI tests (#210) * fix ci ruuner creation * fix python versions * fix setup and test --- .github/workflows/tests.yml | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8bb2491..3d6a9ba 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -27,7 +27,7 @@ jobs: needs: check_code_quality strategy: matrix: - python-version: [3.8, 3.9, 3.10] + python-version: ["3.8", "3.9", "3.10"] os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] runs-on: ${{ matrix.os }} steps: diff --git a/setup.py b/setup.py index 8d15e01..2ece62b 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ extras = {} extras["quality"] = ["black ~= 22.0", "ruff>=0.0.241"] extras["docs_specific"] = ["hf-doc-builder"] extras["dev"] = extras["quality"] + extras["docs_specific"] +extras["test"] = extras["dev"] + ["pytest", "pytest-xdist", "parameterized"] setup( name="peft",