🤗 PEFT
State-of-the-art Parameter-Efficient Fine-Tuning (PEFT) methods
@@ -104,6 +120,9 @@ accelerate launch train_dreambooth.py \ --max_train_steps=800 ``` +Try out the 🤗 Gradio Space which should run seamlessly on a T4 instance: +[smangrul/peft-lora-sd-dreambooth](https://huggingface.co/spaces/smangrul/peft-lora-sd-dreambooth). + ### Parameter Efficient Tuning of LLMs for RLHF components such as Ranker and Policy [ToDo] ### Save compute and storage even for medium and small models diff --git a/setup.py b/setup.py index 2583f08..08188f0 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,8 @@ from setuptools import find_packages extras = {} extras["quality"] = ["black ~= 22.0", "isort >= 5.5.4", "flake8 >= 3.8.3"] -extras["dev"] = extras["quality"] +extras["docs_specific"] = ["hf-doc-builder"] +extras["dev"] = extras["quality"] + extras["docs_specific"] setup( name="peft", diff --git a/src/peft/__init__.py b/src/peft/__init__.py index c1882b3..b76dad8 100644 --- a/src/peft/__init__.py +++ b/src/peft/__init__.py @@ -2,6 +2,21 @@ # There's no way to ignore "F401 '...' imported but unused" warnings in this # module, but to preserve other warnings. So, don't check this module at all. +# coding=utf-8 +# Copyright 2023-present the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + __version__ = "0.1.0.dev0" from .mapping import MODEL_TYPE_TO_PEFT_MODEL_MAPPING, PEFT_TYPE_TO_CONFIG_MAPPING, get_peft_config, get_peft_model diff --git a/src/peft/mapping.py b/src/peft/mapping.py index 9ec027f..b91c7cf 100644 --- a/src/peft/mapping.py +++ b/src/peft/mapping.py @@ -1,3 +1,18 @@ +# coding=utf-8 +# Copyright 2023-present the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from .peft_model import ( PeftModelForCausalLM, PeftModelForSeq2SeqLM, diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py index 2ba74a8..7bf9e15 100644 --- a/src/peft/peft_model.py +++ b/src/peft/peft_model.py @@ -1,3 +1,18 @@ +# coding=utf-8 +# Copyright 2023-present the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import inspect import warnings @@ -20,12 +35,12 @@ class PeftModel(torch.nn.Module): Attributes: - base_model (`PreTrainedModel`): The base transformer model used for Peft. peft_config (`PeftConfig`): - The configuration of the Peft model. modules_to_save (`list` of `str`): The list of sub-module names - to save when saving the model. prompt_encoder (`PromptEncoder`): The prompt encoder used for Peft if - `peft_config.peft_type != PeftType.LORA`. prompt_tokens (`torch.Tensor`): The virtual prompt tokens used for - Peft if `peft_config.peft_type != PeftType.LORA`. transformer_backbone_name (`str`): The name of the - transformer backbone in the base model + base_model (`PreTrainedModel`): The base transformer model used for Peft. peft_config (`PeftConfig`): The + configuration of the Peft model. modules_to_save (`list` of `str`): The list of sub-module names to save when + saving the model. prompt_encoder (`PromptEncoder`): The prompt encoder used for Peft if `peft_config.peft_type + != PeftType.LORA`. prompt_tokens (`torch.Tensor`): The virtual prompt tokens used for Peft if + `peft_config.peft_type != PeftType.LORA`. transformer_backbone_name (`str`): The name of the transformer + backbone in the base model if `peft_config.peft_type != PeftType.LORA`. word_embeddings (`torch.nn.Embedding`): The word embeddings of the transformer backbone in the base model if `peft_config.peft_type != PeftType.LORA`. @@ -151,20 +166,20 @@ class PeftModelForSequenceClassification(PeftModel): peft_config (`PeftConfig`): Peft config. Attributes: - config (`PretrainedConfig`): The configuration object of the base model. cls_layer_name (`str`): The - name of the classification layer. + config (`PretrainedConfig`): The configuration object of the base model. cls_layer_name (`str`): The name of + the classification layer. Example:: >>> from transformers import AutoModelForSequenceClassification >>> from peft import PeftModelForSequenceClassification, get_peft_config >>> config = { - 'peft_type': 'PREFIX_TUNING', 'task_type': 'SEQ_CLS', 'inference_mode': False, 'num_virtual_tokens': 20, - 'token_dim': 768, 'num_transformer_submodules': 1, 'num_attention_heads': 12, 'num_layers': 12, + 'peft_type': 'PREFIX_TUNING', 'task_type': 'SEQ_CLS', 'inference_mode': False, 'num_virtual_tokens': + 20, 'token_dim': 768, 'num_transformer_submodules': 1, 'num_attention_heads': 12, 'num_layers': 12, 'encoder_hidden_size': 768, 'prefix_projection': False, 'postprocess_past_key_value_function': None } - >>> peft_config = get_peft_config(config) - >>> model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased") - >>> peft_model = PeftModelForSequenceClassification(model, peft_config) >>> peft_model.print_trainable_parameters() trainable + >>> peft_config = get_peft_config(config) >>> model = + AutoModelForSequenceClassification.from_pretrained("bert-base-cased") >>> peft_model = + PeftModelForSequenceClassification(model, peft_config) >>> peft_model.print_trainable_parameters() trainable params: 370178 || all params: 108680450 || trainable%: 0.3406113979101117 """ @@ -323,17 +338,15 @@ class PeftModelForCausalLM(PeftModel): Example:: - >>> from transformers import AutoModelForCausalLM >>> from peft import PeftModelForCausalLM, get_peft_config >>> - config = { + >>> from transformers import AutoModelForCausalLM >>> from peft import PeftModelForCausalLM, get_peft_config + >>> config = { 'peft_type': 'PREFIX_TUNING', 'task_type': 'CAUSAL_LM', 'inference_mode': False, 'num_virtual_tokens': 20, 'token_dim': 1280, 'num_transformer_submodules': 1, 'num_attention_heads': 20, 'num_layers': 36, 'encoder_hidden_size': 1280, 'prefix_projection': False, 'postprocess_past_key_value_function': None } - >>> peft_config = get_peft_config(config) - >>> model = AutoModelForCausalLM.from_pretrained("gpt2-large") - >>> peft_model = PeftModelForCausalLM(model, peft_config) - >>> peft_model.print_trainable_parameters() trainable params: - 1843200 || all params: 775873280 || trainable%: 0.23756456724479544 + >>> peft_config = get_peft_config(config) >>> model = AutoModelForCausalLM.from_pretrained("gpt2-large") >>> + peft_model = PeftModelForCausalLM(model, peft_config) >>> peft_model.print_trainable_parameters() trainable + params: 1843200 || all params: 775873280 || trainable%: 0.23756456724479544 """ def __init__(self, model, peft_config: PeftConfig): @@ -447,16 +460,14 @@ class PeftModelForSeq2SeqLM(PeftModel): Example:: - >>> from transformers import AutoModelForSeq2SeqLM >>> from peft import PeftModelForSeq2SeqLM, get_peft_config >>> - config = { + >>> from transformers import AutoModelForSeq2SeqLM >>> from peft import PeftModelForSeq2SeqLM, get_peft_config + >>> config = { 'peft_type': 'LORA', 'task_type': 'SEQ_2_SEQ_LM', 'inference_mode': False, 'r': 8, 'target_modules': ['q', 'v'], 'lora_alpha': 32, 'lora_dropout': 0.1, 'merge_weights': False, 'fan_in_fan_out': False, 'enable_lora': None, 'bias': 'none' } - >>> peft_config = get_peft_config(config) - >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") - >>> peft_model = PeftModelForSeq2SeqLM(model, peft_config) - >>> peft_model.print_trainable_parameters() trainable + >>> peft_config = get_peft_config(config) >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") >>> + peft_model = PeftModelForSeq2SeqLM(model, peft_config) >>> peft_model.print_trainable_parameters() trainable params: 884736 || all params: 223843584 || trainable%: 0.3952474242013566 """ @@ -599,8 +610,8 @@ class PeftModelForTokenClassification(PeftModel): peft_config (`PeftConfig`): Peft config. Attributes: - config (`PretrainedConfig`): The configuration object of the base model. cls_layer_name (`str`): The - name of the classification layer. + config (`PretrainedConfig`): The configuration object of the base model. cls_layer_name (`str`): The name of + the classification layer. Example:: @@ -611,9 +622,8 @@ class PeftModelForTokenClassification(PeftModel): 'encoder_hidden_size': 768, 'prefix_projection': False, 'postprocess_past_key_value_function': None } >>> peft_config = get_peft_config(config) >>> model = - AutoModelForTokenClassification.from_pretrained("bert-base-cased") - >>> peft_model = PeftModelForTokenClassification(model, peft_config) - >>> peft_model.print_trainable_parameters() trainable + AutoModelForTokenClassification.from_pretrained("bert-base-cased") >>> peft_model = + PeftModelForTokenClassification(model, peft_config) >>> peft_model.print_trainable_parameters() trainable params: 370178 || all params: 108680450 || trainable%: 0.3406113979101117 """ diff --git a/src/peft/tuners/__init__.py b/src/peft/tuners/__init__.py index 54e554c..38b7926 100644 --- a/src/peft/tuners/__init__.py +++ b/src/peft/tuners/__init__.py @@ -2,6 +2,21 @@ # There's no way to ignore "F401 '...' imported but unused" warnings in this # module, but to preserve other warnings. So, don't check this module at all +# coding=utf-8 +# Copyright 2023-present the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from .lora import LoraConfig, LoraModel from .p_tuning import PromptEncoder, PromptEncoderConfig, PromptEncoderReparameterizationType from .prefix_tuning import PrefixEncoder, PrefixTuningConfig diff --git a/src/peft/tuners/lora.py b/src/peft/tuners/lora.py index 9697152..e3ded08 100644 --- a/src/peft/tuners/lora.py +++ b/src/peft/tuners/lora.py @@ -1,3 +1,18 @@ +# coding=utf-8 +# Copyright 2023-present the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import math import warnings from dataclasses import asdict, dataclass, field @@ -70,8 +85,8 @@ class LoraModel(torch.nn.Module): >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") >>> lora_model = LoraModel(config, model) Attributes: - model (`transformers.PreTrainedModel`): The model to be adapted. config (`LoraConfig`): The - configuration of the Lora model. + model (`transformers.PreTrainedModel`): The model to be adapted. config (`LoraConfig`): The configuration of + the Lora model. """ def __init__(self, config, model): @@ -237,7 +252,7 @@ class Linear(nn.Linear, LoraLayer): result += self.lora_B(self.lora_A(self.lora_dropout(x))) * self.scaling return result else: - return F.linear(x, T(self.weight), bias=self.bias) + return F.linear(x, transpose(self.weight, self.fan_in_fan_out), bias=self.bias) class MergedLinear(nn.Linear, LoraLayer): diff --git a/src/peft/tuners/p_tuning.py b/src/peft/tuners/p_tuning.py index d5cca27..98f6537 100644 --- a/src/peft/tuners/p_tuning.py +++ b/src/peft/tuners/p_tuning.py @@ -1,3 +1,18 @@ +# coding=utf-8 +# Copyright 2023-present the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import enum from dataclasses import dataclass, field from typing import Union @@ -19,8 +34,7 @@ class PromptEncoderConfig(PromptLearningConfig): Args: encoder_reparameterization_type - (Union[:class:`PromptEncoderReparameterizationType`, `str`]): The type of reparameterization to - use. + (Union[:class:`PromptEncoderReparameterizationType`, `str`]): The type of reparameterization to use. encoder_hidden_size (`int`): The hidden size of the prompt encoder. encoder_num_layers (`int`): The number of layers of the prompt encoder. encoder_dropout (`float`): The dropout probability of the prompt encoder. @@ -70,11 +84,10 @@ class PromptEncoder(torch.nn.Module): (:class:`~torch.nn.Sequential`): The MLP head of the prompt encoder if `inference_mode=False`. lstm_head (:class:`~torch.nn.LSTM`): The LSTM head of the prompt encoder if `inference_mode=False` and `encoder_reparameterization_type="LSTM"`. - token_dim (`int`): The hidden embedding dimension of the base transformer model. input_size (`int`): - The input size of the prompt encoder. output_size (`int`): The output size of the prompt encoder. - hidden_size (`int`): The hidden size of the prompt encoder. total_virtual_tokens (`int`): The total - number of virtual tokens of the prompt encoder. encoder_type - (Union[:class:`PromptEncoderReparameterizationType`, `str`]): + token_dim (`int`): The hidden embedding dimension of the base transformer model. input_size (`int`): The input + size of the prompt encoder. output_size (`int`): The output size of the prompt encoder. hidden_size (`int`): + The hidden size of the prompt encoder. total_virtual_tokens (`int`): The total number of virtual tokens of the + prompt encoder. encoder_type (Union[:class:`PromptEncoderReparameterizationType`, `str`]): The encoder type of the prompt encoder. diff --git a/src/peft/tuners/prefix_tuning.py b/src/peft/tuners/prefix_tuning.py index e20a49c..48ae168 100644 --- a/src/peft/tuners/prefix_tuning.py +++ b/src/peft/tuners/prefix_tuning.py @@ -1,3 +1,19 @@ +# coding=utf-8 +# Copyright 2023-present the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + from dataclasses import dataclass, field from typing import Callable, Optional @@ -54,8 +70,9 @@ class PrefixEncoder(torch.nn.Module): Attributes: - embedding (`torch.nn.Embedding`): The embedding layer of the prefix encoder. trans - (`torch.nn.Sequential`): The two-layer MLP to transform the prefix embeddings + embedding (`torch.nn.Embedding`): + The embedding layer of the prefix encoder. trans (`torch.nn.Sequential`): The + two-layer MLP to transform the prefix embeddings if `prefix_projection` is `True`. prefix_projection (`bool`): Whether to project the prefix embeddings. diff --git a/src/peft/tuners/prompt_tuning.py b/src/peft/tuners/prompt_tuning.py index 3717964..04e2ef2 100644 --- a/src/peft/tuners/prompt_tuning.py +++ b/src/peft/tuners/prompt_tuning.py @@ -1,3 +1,18 @@ +# coding=utf-8 +# Copyright 2023-present the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import enum import math from dataclasses import dataclass, field diff --git a/src/peft/utils/__init__.py b/src/peft/utils/__init__.py index 54b1ef0..45b0cd5 100644 --- a/src/peft/utils/__init__.py +++ b/src/peft/utils/__init__.py @@ -2,6 +2,21 @@ # There's no way to ignore "F401 '...' imported but unused" warnings in this # module, but to preserve other warnings. So, don't check this module at all +# coding=utf-8 +# Copyright 2023-present the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from .config import PeftConfig, PeftType, PromptLearningConfig, TaskType from .other import _set_trainable, bloom_model_postprocess_past_key_value, shift_tokens_right, transpose from .save_and_load import get_peft_model_state_dict, peft_model_load_and_dispatch, set_peft_model_state_dict diff --git a/src/peft/utils/config.py b/src/peft/utils/config.py index 6c041e7..8e485eb 100644 --- a/src/peft/utils/config.py +++ b/src/peft/utils/config.py @@ -1,3 +1,18 @@ +# coding=utf-8 +# Copyright 2023-present the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import enum from dataclasses import dataclass, field from typing import Optional, Union diff --git a/src/peft/utils/other.py b/src/peft/utils/other.py index 4063e59..14ab90e 100644 --- a/src/peft/utils/other.py +++ b/src/peft/utils/other.py @@ -1,3 +1,18 @@ +# coding=utf-8 +# Copyright 2023-present the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import torch diff --git a/src/peft/utils/save_and_load.py b/src/peft/utils/save_and_load.py index 36867fd..f4062af 100644 --- a/src/peft/utils/save_and_load.py +++ b/src/peft/utils/save_and_load.py @@ -1,3 +1,18 @@ +# coding=utf-8 +# Copyright 2023-present the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from .config import PeftType diff --git a/utils/style_doc.py b/utils/style_doc.py deleted file mode 100644 index 0422ebe..0000000 --- a/utils/style_doc.py +++ /dev/null @@ -1,556 +0,0 @@ -# coding=utf-8 -# Copyright 2020 The HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Style utils for the .rst and the docstrings.""" - -import argparse -import os -import re -import warnings - -import black - - -BLACK_AVOID_PATTERNS = {} - - -# Regexes -# Re pattern that catches list introduction (with potential indent) -_re_list = re.compile(r"^(\s*-\s+|\s*\*\s+|\s*\d+\.\s+)") -# Re pattern that catches code block introduction (with potential indent) -_re_code = re.compile(r"^(\s*)```(.*)$") -# Re pattern that catches rst args blocks of the form `Parameters:`. -_re_args = re.compile("^\s*(Args?|Arguments?|Params?|Parameters?):\s*$") -# Re pattern that catches return blocks of the form `Return:`. -_re_returns = re.compile("^\s*Returns?:\s*$") -# Matches the special tag to ignore some paragraphs. -_re_doc_ignore = re.compile(r"(\.\.|#)\s*docstyle-ignore") -# Re pattern that matches