From 39fb96316fa7c72cdfe95b9b7b63ff71fe85b942 Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 30 Mar 2023 14:16:01 -0700 Subject: [PATCH 1/6] first draft of api docs --- docs/source/_toctree.yml | 3 +- docs/source/package_reference/config.mdx | 19 +++++++++- docs/source/package_reference/peft_model.mdx | 37 +++++++++++++++++++- docs/source/package_reference/tuners.mdx | 36 ++++++++++++++++++- 4 files changed, 91 insertions(+), 4 deletions(-) diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index 1255844..12f8901 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -19,4 +19,5 @@ - local: package_reference/config title: Configuration - local: package_reference/tuners - title: Tuners \ No newline at end of file + title: Tuners + diff --git a/docs/source/package_reference/config.mdx b/docs/source/package_reference/config.mdx index af4abbf..1a2212f 100644 --- a/docs/source/package_reference/config.mdx +++ b/docs/source/package_reference/config.mdx @@ -1 +1,18 @@ -# Configuration \ No newline at end of file +# Configuration + +The configuration classes stores the configuration of a [`PeftModel`], PEFT adapter models, and the configurations of [`PrefixTuning`], [`PromptTuning`], and [`PromptEncoder`]. They contain methods for saving and loading model configurations from the Hub, specifying the PEFT method to use, type of task to perform, and model configurations like number of layers and number of attention heads. + +## PeftConfigMixin + +[[autodoc]] PeftConfigMixin + - all + +## PeftConfig + +[[autodoc]] PeftConfig + - all + +## PromptLearningConfig + +[[autodoc]] PromptLearningConfig + - all diff --git a/docs/source/package_reference/peft_model.mdx b/docs/source/package_reference/peft_model.mdx index f6789d4..771fd49 100644 --- a/docs/source/package_reference/peft_model.mdx +++ b/docs/source/package_reference/peft_model.mdx @@ -1 +1,36 @@ -# PEFT model \ No newline at end of file +## Models + +[`PeftModel`] is the base model class for specifying the base Transformer model and configuration to apply a PEFT method to. The base `PeftModel` contains methods for loading and saving models from the Hub, and supports the [`PromptEncoder`] for prompt learning. + +## PeftModel + +[[autodoc]] PeftModel + - all + +## PeftModelForSequenceClassification + +A `PeftModel` for sequence classification tasks. + +[[autodoc]] PeftModelForSequenceClassification + - all + +## PeftModelForTokenClassification + +A `PeftModel` for token classification tasks. + +[[autodoc]] PeftModelForTokenClassification + - all + +## PeftModelForCausalLM + +A `PeftModel` for causal language modeling. + +[[autodoc]] PeftModelForCausalLM + - all + +## PeftModelForSeq2SeqLM + +A `PeftModel` for sequence-to-sequence language modeling. + +[[autodoc]] PeftModelForSeq2SeqLM + - all diff --git a/docs/source/package_reference/tuners.mdx b/docs/source/package_reference/tuners.mdx index 1e60f16..93c0dc6 100644 --- a/docs/source/package_reference/tuners.mdx +++ b/docs/source/package_reference/tuners.mdx @@ -1 +1,35 @@ -# Tuners \ No newline at end of file +# Tuners + +Each tuner (or PEFT method) has a configuration and model. + +## LoRA + +For finetuning a model with LoRA. + +[[autodoc]] LoraConfig + +[[autodoc]] LoraModel + +[[autodoc]] LoraLayer + +[[autodoc]] Linear + +[[autodoc]] MergedLinear + +## P-tuning + +[[autodoc]] PromptEncoderConfig + +[[autodoc]] PromptEncoder + +## Prefix tuning + +[[autodoc]] PrefixTuningConfig + +[[autodoc]] PrefixEncoder + +## Prompt tuning + +[[autodoc]] PromptTuningConfig + +[[autodoc]] PromptEmbedding From 8fd53e004518cb52a7def340adfd0e873ae6298d Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 30 Mar 2023 14:26:12 -0700 Subject: [PATCH 2/6] fix path to peftconfigmixin? --- docs/source/package_reference/config.mdx | 2 +- docs/source/package_reference/peft_model.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/package_reference/config.mdx b/docs/source/package_reference/config.mdx index 1a2212f..6866685 100644 --- a/docs/source/package_reference/config.mdx +++ b/docs/source/package_reference/config.mdx @@ -4,7 +4,7 @@ The configuration classes stores the configuration of a [`PeftModel`], PEFT adap ## PeftConfigMixin -[[autodoc]] PeftConfigMixin +[[autodoc]] utils.config.PeftConfigMixin - all ## PeftConfig diff --git a/docs/source/package_reference/peft_model.mdx b/docs/source/package_reference/peft_model.mdx index 771fd49..f2618ef 100644 --- a/docs/source/package_reference/peft_model.mdx +++ b/docs/source/package_reference/peft_model.mdx @@ -1,4 +1,4 @@ -## Models +# Models [`PeftModel`] is the base model class for specifying the base Transformer model and configuration to apply a PEFT method to. The base `PeftModel` contains methods for loading and saving models from the Hub, and supports the [`PromptEncoder`] for prompt learning. From 47f05fe7b574130ba91beda623e0f9a8d261d243 Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 30 Mar 2023 14:31:57 -0700 Subject: [PATCH 3/6] fix path to loralayer too --- docs/source/package_reference/tuners.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/package_reference/tuners.mdx b/docs/source/package_reference/tuners.mdx index 93c0dc6..a44520c 100644 --- a/docs/source/package_reference/tuners.mdx +++ b/docs/source/package_reference/tuners.mdx @@ -10,7 +10,7 @@ For finetuning a model with LoRA. [[autodoc]] LoraModel -[[autodoc]] LoraLayer +[[autodoc]] tuners.lora.LoraLayer [[autodoc]] Linear From 7c31f5156723e56defae88fb760a193205a0b676 Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 30 Mar 2023 14:46:48 -0700 Subject: [PATCH 4/6] use explicit path --- docs/source/package_reference/tuners.mdx | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/source/package_reference/tuners.mdx b/docs/source/package_reference/tuners.mdx index a44520c..2ec0824 100644 --- a/docs/source/package_reference/tuners.mdx +++ b/docs/source/package_reference/tuners.mdx @@ -12,24 +12,24 @@ For finetuning a model with LoRA. [[autodoc]] tuners.lora.LoraLayer -[[autodoc]] Linear +[[autodoc]] tuners.lora.Linear -[[autodoc]] MergedLinear +[[autodoc]] tuners.lora.MergedLinear ## P-tuning -[[autodoc]] PromptEncoderConfig +[[autodoc]] tuners.p_tuning.PromptEncoderConfig -[[autodoc]] PromptEncoder +[[autodoc]] tuners.p_tuning.PromptEncoder ## Prefix tuning -[[autodoc]] PrefixTuningConfig +[[autodoc]] tuners.prefix_tuning.PrefixTuningConfig -[[autodoc]] PrefixEncoder +[[autodoc]] tuners.prefix_tuning.PrefixEncoder ## Prompt tuning -[[autodoc]] PromptTuningConfig +[[autodoc]] tuners.prompt_tuning.PromptTuningConfig -[[autodoc]] PromptEmbedding +[[autodoc]] tuners.prompt_tuning.PromptEmbedding \ No newline at end of file From 622a5a231ef36d2fa114032bcfe130c6299f9a24 Mon Sep 17 00:00:00 2001 From: Steven Liu Date: Fri, 31 Mar 2023 14:30:05 -0700 Subject: [PATCH 5/6] clean up docstrings --- src/peft/peft_model.py | 194 ++++++++++++++++++++----------- src/peft/tuners/lora.py | 39 ++++--- src/peft/tuners/p_tuning.py | 45 ++++--- src/peft/tuners/prefix_tuning.py | 36 +++--- src/peft/tuners/prompt_tuning.py | 44 ++++--- src/peft/utils/config.py | 12 +- 6 files changed, 235 insertions(+), 135 deletions(-) diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py index 7491342..2b79f4c 100644 --- a/src/peft/peft_model.py +++ b/src/peft/peft_model.py @@ -45,26 +45,26 @@ from .utils import ( class PeftModel(PushToHubMixin, torch.nn.Module): """ - Parameter-Efficient Fine-Tuning Model. Base model encompassing various Peft methods. + Base model encompassing various Peft methods. Args: - model ([`PreTrainedModel`]): The base transformer model used for Peft. + model ([`~transformers.PreTrainedModel`]): The base transformer model used for Peft. peft_config ([`PeftConfig`]): The configuration of the Peft model. **Attributes**: - - **base_model** ([`PreTrainedModel`]) -- The base transformer model used for Peft. + - **base_model** ([`~transformers.PreTrainedModel`]) -- The base transformer model used for Peft. - **peft_config** ([`PeftConfig`]) -- The configuration of the Peft model. - **modules_to_save** (`list` of `str`) -- The list of sub-module names to save when saving the model. - **prompt_encoder** ([`PromptEncoder`]) -- The prompt encoder used for Peft if - `isinstance(self.peft_config, PromptLearningConfig)`. + using [`PromptLearningConfig`]. - **prompt_tokens** (`torch.Tensor`) -- The virtual prompt tokens used for Peft if - `isinstance(self.peft_config, PromptLearningConfig)`. + using [`PromptLearningConfig`]. - **transformer_backbone_name** (`str`) -- The name of the transformer - backbone in the base model if `isinstance(self.peft_config, PromptLearningConfig)`. + backbone in the base model if using [`PromptLearningConfig`]. - **word_embeddings** (`torch.nn.Embedding`) -- The word embeddings of the transformer backbone - in the base model if `isinstance(self.peft_config, PromptLearningConfig)`. + in the base model if using [`PromptLearningConfig`]. """ def __init__(self, model, peft_config: PeftConfig): @@ -84,10 +84,11 @@ class PeftModel(PushToHubMixin, torch.nn.Module): def save_pretrained(self, save_directory, **kwargs): r""" - Args: This function saves the adapter model and the adapter configuration files to a directory, so that it can be - re-loaded using the `LoraModel.from_pretrained` class method, and also used by the `LoraModel.push_to_hub` + reloaded using the [`LoraModel.from_pretrained`] class method, and also used by the [`LoraModel.push_to_hub`] method. + + Args: save_directory (`str`): Directory where the adapter model and configuration files will be saved (will be created if it does not exist). @@ -117,17 +118,18 @@ class PeftModel(PushToHubMixin, torch.nn.Module): @classmethod def from_pretrained(cls, model, model_id, **kwargs): r""" + Instantiate a [`LoraModel`] from a pretrained Lora configuration and weights. + Args: - Instantiate a `LoraModel` from a pretrained Lora configuration and weights. - model (`transformers.PreTrainedModel`): - The model to be adapted. The model should be initialized with the `from_pretrained` method. from - `transformers` library. - model_id (`str`): + model ([`~transformers.PreTrainedModel`]): + The model to be adapted. The model should be initialized with the + [`~transformers.PreTrainedModel.from_pretrained`] method from the 🤗 Transformers library. + model_id (`str` or `os.PathLike`): The name of the Lora configuration to use. Can be either: - - A string, the `model id` of a Lora configuration hosted inside a model repo on - huggingface Hub - - A path to a directory containing a Lora configuration file saved using the - `save_pretrained` method, e.g., ``./my_lora_config_directory/``. + - A string, the `model id` of a Lora configuration hosted inside a model repo on the Hugging Face + Hub. + - A path to a directory containing a Lora configuration file saved using the `save_pretrained` + method (`./my_lora_config_directory/`). """ from .mapping import MODEL_TYPE_TO_PEFT_MODEL_MAPPING, PEFT_TYPE_TO_CONFIG_MAPPING @@ -322,25 +324,39 @@ class PeftModelForSequenceClassification(PeftModel): Peft model for sequence classification tasks. Args: - model ([`PreTrainedModel`]): Base transformer model + model ([`~transformers.PreTrainedModel`]): Base transformer model. peft_config ([`PeftConfig`]): Peft config. **Attributes**: - - **config** ([`PretrainedConfig`]) -- The configuration object of the base model. + - **config** ([`~transformers.PretrainedConfig`]) -- The configuration object of the base model. - **cls_layer_name** (`str`) -- The name of the classification layer. - Example:: + Example: - >>> from transformers import AutoModelForSequenceClassification >>> from peft import - PeftModelForSequenceClassification, get_peft_config >>> config = { - 'peft_type': 'PREFIX_TUNING', 'task_type': 'SEQ_CLS', 'inference_mode': False, 'num_virtual_tokens': - 20, 'token_dim': 768, 'num_transformer_submodules': 1, 'num_attention_heads': 12, 'num_layers': 12, - 'encoder_hidden_size': 768, 'prefix_projection': False, 'postprocess_past_key_value_function': None - } - >>> peft_config = get_peft_config(config) >>> model = - AutoModelForSequenceClassification.from_pretrained("bert-base-cased") >>> peft_model = - PeftModelForSequenceClassification(model, peft_config) >>> peft_model.print_trainable_parameters() trainable - params: 370178 || all params: 108680450 || trainable%: 0.3406113979101117 + ```py + >>> from transformers import AutoModelForSequenceClassification + >>> from peft import PeftModelForSequenceClassification, get_peft_config + + >>> config = { + ... "peft_type": "PREFIX_TUNING", + ... "task_type": "SEQ_CLS", + ... "inference_mode": False, + ... "num_virtual_tokens": 20, + ... "token_dim": 768, + ... "num_transformer_submodules": 1, + ... "num_attention_heads": 12, + ... "num_layers": 12, + ... "encoder_hidden_size": 768, + ... "prefix_projection": False, + ... "postprocess_past_key_value_function": None, + ... } + + >>> peft_config = get_peft_config(config) + >>> model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased") + >>> peft_model = PeftModelForSequenceClassification(model, peft_config) + >>> peft_model.print_trainable_parameters() + trainable params: 370178 || all params: 108680450 || trainable%: 0.3406113979101117 + ``` """ def __init__(self, model, peft_config: PeftConfig): @@ -490,24 +506,39 @@ class PeftModelForSequenceClassification(PeftModel): class PeftModelForCausalLM(PeftModel): """ - Peft model for Causal LM + Peft model for causal language modeling. Args: - model ([`PreTrainedModel`]): Base transformer model + model ([`~transformers.PreTrainedModel`]): Base transformer model. peft_config ([`PeftConfig`]): Peft config. - Example:: + Example: + + ```py + >>> from transformers import AutoModelForCausalLM + >>> from peft import PeftModelForCausalLM, get_peft_config - >>> from transformers import AutoModelForCausalLM >>> from peft import PeftModelForCausalLM, get_peft_config >>> config = { - 'peft_type': 'PREFIX_TUNING', 'task_type': 'CAUSAL_LM', 'inference_mode': False, 'num_virtual_tokens': - 20, 'token_dim': 1280, 'num_transformer_submodules': 1, 'num_attention_heads': 20, 'num_layers': 36, - 'encoder_hidden_size': 1280, 'prefix_projection': False, 'postprocess_past_key_value_function': None - } - >>> peft_config = get_peft_config(config) >>> model = AutoModelForCausalLM.from_pretrained("gpt2-large") >>> - peft_model = PeftModelForCausalLM(model, peft_config) >>> peft_model.print_trainable_parameters() trainable - params: 1843200 || all params: 775873280 || trainable%: 0.23756456724479544 + ... "peft_type": "PREFIX_TUNING", + ... "task_type": "CAUSAL_LM", + ... "inference_mode": False, + ... "num_virtual_tokens": 20, + ... "token_dim": 1280, + ... "num_transformer_submodules": 1, + ... "num_attention_heads": 20, + ... "num_layers": 36, + ... "encoder_hidden_size": 1280, + ... "prefix_projection": False, + ... "postprocess_past_key_value_function": None, + ... } + + >>> peft_config = get_peft_config(config) + >>> model = AutoModelForCausalLM.from_pretrained("gpt2-large") + >>> peft_model = PeftModelForCausalLM(model, peft_config) + >>> peft_model.print_trainable_parameters() + trainable params: 1843200 || all params: 775873280 || trainable%: 0.23756456724479544 + ``` """ def __init__(self, model, peft_config: PeftConfig): @@ -641,24 +672,39 @@ class PeftModelForCausalLM(PeftModel): class PeftModelForSeq2SeqLM(PeftModel): """ - Peft model for Seq2Seq LM + Peft model for sequence-to-sequence language modeling. Args: - model ([`PreTrainedModel`]): Base transformer model + model ([`~transformers.PreTrainedModel`]): Base transformer model. peft_config ([`PeftConfig`]): Peft config. - Example:: + Example: + + ```py + >>> from transformers import AutoModelForSeq2SeqLM + >>> from peft import PeftModelForSeq2SeqLM, get_peft_config - >>> from transformers import AutoModelForSeq2SeqLM >>> from peft import PeftModelForSeq2SeqLM, get_peft_config >>> config = { - 'peft_type': 'LORA', 'task_type': 'SEQ_2_SEQ_LM', 'inference_mode': False, 'r': 8, 'target_modules': - ['q', 'v'], 'lora_alpha': 32, 'lora_dropout': 0.1, 'merge_weights': False, 'fan_in_fan_out': False, - 'enable_lora': None, 'bias': 'none' - } - >>> peft_config = get_peft_config(config) >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") >>> - peft_model = PeftModelForSeq2SeqLM(model, peft_config) >>> peft_model.print_trainable_parameters() trainable - params: 884736 || all params: 223843584 || trainable%: 0.3952474242013566 + ... "peft_type": "LORA", + ... "task_type": "SEQ_2_SEQ_LM", + ... "inference_mode": False, + ... "r": 8, + ... "target_modules": ["q", "v"], + ... "lora_alpha": 32, + ... "lora_dropout": 0.1, + ... "merge_weights": False, + ... "fan_in_fan_out": False, + ... "enable_lora": None, + ... "bias": "none", + ... } + + >>> peft_config = get_peft_config(config) + >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") + >>> peft_model = PeftModelForSeq2SeqLM(model, peft_config) + >>> peft_model.print_trainable_parameters() + trainable params: 884736 || all params: 223843584 || trainable%: 0.3952474242013566 + ``` """ def __init__(self, model, peft_config: PeftConfig): @@ -808,28 +854,42 @@ class PeftModelForSeq2SeqLM(PeftModel): class PeftModelForTokenClassification(PeftModel): """ - Peft model for sequence classification tasks. + Peft model for token classification tasks. Args: - model ([`PreTrainedModel`]): Base transformer model + model ([`~transformers.PreTrainedModel`]): Base transformer model. peft_config ([`PeftConfig`]): Peft config. **Attributes**: - - **config** ([`PretrainedConfig`]) -- The configuration object of the base model. + - **config** ([`~transformers.PretrainedConfig`]) -- The configuration object of the base model. - **cls_layer_name** (`str`) -- The name of the classification layer. - Example:: + Example: - >>> from transformers import AutoModelForSequenceClassification >>> from peft import - PeftModelForTokenClassification, get_peft_config >>> config = { - 'peft_type': 'PREFIX_TUNING', 'task_type': 'TOKEN_CLS', 'inference_mode': False, 'num_virtual_tokens': - 20, 'token_dim': 768, 'num_transformer_submodules': 1, 'num_attention_heads': 12, 'num_layers': 12, - 'encoder_hidden_size': 768, 'prefix_projection': False, 'postprocess_past_key_value_function': None - } - >>> peft_config = get_peft_config(config) >>> model = - AutoModelForTokenClassification.from_pretrained("bert-base-cased") >>> peft_model = - PeftModelForTokenClassification(model, peft_config) >>> peft_model.print_trainable_parameters() trainable - params: 370178 || all params: 108680450 || trainable%: 0.3406113979101117 + ```py + >>> from transformers import AutoModelForSequenceClassification + >>> from peft import PeftModelForTokenClassification, get_peft_config + + >>> config = { + ... "peft_type": "PREFIX_TUNING", + ... "task_type": "TOKEN_CLS", + ... "inference_mode": False, + ... "num_virtual_tokens": 20, + ... "token_dim": 768, + ... "num_transformer_submodules": 1, + ... "num_attention_heads": 12, + ... "num_layers": 12, + ... "encoder_hidden_size": 768, + ... "prefix_projection": False, + ... "postprocess_past_key_value_function": None, + ... } + + >>> peft_config = get_peft_config(config) + >>> model = AutoModelForTokenClassification.from_pretrained("bert-base-cased") + >>> peft_model = PeftModelForTokenClassification(model, peft_config) + >>> peft_model.print_trainable_parameters() + trainable params: 370178 || all params: 108680450 || trainable%: 0.3406113979101117 + ``` """ def __init__(self, model, peft_config: PeftConfig): diff --git a/src/peft/tuners/lora.py b/src/peft/tuners/lora.py index 47f2c02..51cd56f 100644 --- a/src/peft/tuners/lora.py +++ b/src/peft/tuners/lora.py @@ -39,19 +39,19 @@ if is_bnb_available(): @dataclass class LoraConfig(PeftConfig): """ - This is the configuration class to store the configuration of a [`~peft.Lora`]. + This is the configuration class to store the configuration of a [`LoraModel`]. Args: - r (`int`): Lora attention dimension + r (`int`): Lora attention dimension. target_modules (`Union[List[str],str]`): The names of the modules to apply Lora to. lora_alpha (`float`): The alpha parameter for Lora scaling. lora_dropout (`float`): The dropout probability for Lora layers. merge_weights (`bool`): Whether to merge the weights of the Lora layers with the base transformer model in `eval` mode. - fan_in_fan_out (`bool`): Set this to True if the layer to replace stores weight like (fan_in, fan_out) - enable_lora ( `List[bool]`): Used with `lora.MergedLinear`. - bias (`str`): Bias type for Lora. Can be 'none', 'all' or 'lora_only' - modules_to_save (`List[str]`):List of modules apart from LoRA layers to be set as trainable + fan_in_fan_out (`bool`): Set this to True if the layer to replace stores weight like (`fan_in`, `fan_out`). + enable_lora ( `List[bool]`): Used with [`lora.MergedLinear`]. + bias (`str`): Bias type for Lora. Can be `none`, `all` or `lora_only`. + modules_to_save (`List[str]`): List of modules apart from Lora layers to be set as trainable and saved in the final checkpoint. """ @@ -96,22 +96,33 @@ class LoraModel(torch.nn.Module): Creates Low Rank Adapter (Lora) model from a pretrained transformers model. Args: - model ([`transformers.PreTrainedModel`]): The model to be adapted. + model ([`~transformers.PreTrainedModel`]): The model to be adapted. config ([`LoraConfig`]): The configuration of the Lora model. Returns: `torch.nn.Module`: The Lora model. - Example:: + Example: - >>> from transformers import AutoModelForSeq2SeqLM, LoraConfig >>> from peft import LoraModel, LoraConfig >>> - config = LoraConfig( - peft_type="LORA", task_type="SEQ_2_SEQ_LM", r=8, lora_alpha=32, target_modules=["q", "v"], - lora_dropout=0.01, ) - >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") >>> lora_model = LoraModel(config, model) + ```py + >>> from transformers import AutoModelForSeq2SeqLM, LoraConfig + >>> from peft import LoraModel, LoraConfig + + >>> config = LoraConfig( + ... peft_type="LORA", + ... task_type="SEQ_2_SEQ_LM", + ... r=8, + ... lora_alpha=32, + ... target_modules=["q", "v"], + ... lora_dropout=0.01, + ... ) + + >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") + >>> lora_model = LoraModel(config, model) + ``` **Attributes**: - - **model** ([`transformers.PreTrainedModel`]) -- The model to be adapted. + - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted. - **peft_config** ([`LoraConfig`]): The configuration of the Lora model. """ diff --git a/src/peft/tuners/p_tuning.py b/src/peft/tuners/p_tuning.py index b9c38c4..4a272f3 100644 --- a/src/peft/tuners/p_tuning.py +++ b/src/peft/tuners/p_tuning.py @@ -31,11 +31,11 @@ class PromptEncoderReparameterizationType(str, enum.Enum): @dataclass class PromptEncoderConfig(PromptLearningConfig): """ - This is the configuration class to store the configuration of a [`~peft.PromptEncoder`]. + This is the configuration class to store the configuration of a [`PromptEncoder`]. Args: - encoder_reparameterization_type - (Union[[`PromptEncoderReparameterizationType`], `str`]): The type of reparameterization to use. + encoder_reparameterization_type (Union[[`PromptEncoderReparameterizationType`], `str`]): + The type of reparameterization to use. encoder_hidden_size (`int`): The hidden size of the prompt encoder. encoder_num_layers (`int`): The number of layers of the prompt encoder. encoder_dropout (`float`): The dropout probability of the prompt encoder. @@ -71,19 +71,30 @@ class PromptEncoder(torch.nn.Module): Args: config ([`PromptEncoderConfig`]): The configuration of the prompt encoder. - Example:: + Example: - >>> from peft import PromptEncoder, PromptEncoderConfig >>> config = PromptEncoderConfig( - peft_type="P_TUNING", task_type="SEQ_2_SEQ_LM", num_virtual_tokens=20, token_dim=768, - num_transformer_submodules=1, num_attention_heads=12, num_layers=12, - encoder_reparameterization_type="MLP", encoder_hidden_size=768 - ) - >>> prompt_encoder = PromptEncoder(config) + ```py + >>> from peft import PromptEncoder, PromptEncoderConfig + + >>> config = PromptEncoderConfig( + ... peft_type="P_TUNING", + ... task_type="SEQ_2_SEQ_LM", + ... num_virtual_tokens=20, + ... token_dim=768, + ... num_transformer_submodules=1, + ... num_attention_heads=12, + ... num_layers=12, + ... encoder_reparameterization_type="MLP", + ... encoder_hidden_size=768, + ... ) + + >>> prompt_encoder = PromptEncoder(config) + ``` **Attributes**: - - **embedding** ([`~torch.nn.Embedding`]) -- The embedding layer of the prompt encoder. - - **mlp_head** ([`~torch.nn.Sequential`]) -- The MLP head of the prompt encoder if `inference_mode=False`. - - **lstm_head** ([`~torch.nn.LSTM`]) -- The LSTM head of the prompt encoder if `inference_mode=False` and + - **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prompt encoder. + - **mlp_head** (`torch.nn.Sequential`) -- The MLP head of the prompt encoder if `inference_mode=False`. + - **lstm_head** (`torch.nn.LSTM`) -- The LSTM head of the prompt encoder if `inference_mode=False` and `encoder_reparameterization_type="LSTM"`. - **token_dim** (`int`) -- The hidden embedding dimension of the base transformer model. - **input_size** (`int`) -- The input size of the prompt encoder. @@ -91,13 +102,13 @@ class PromptEncoder(torch.nn.Module): - **hidden_size** (`int`) -- The hidden size of the prompt encoder. - **total_virtual_tokens** (`int`): The total number of virtual tokens of the prompt encoder. - - **encoder_type** (Union[[`PromptEncoderReparameterizationType`], `str`]): - The encoder type of the prompt encoder. + - **encoder_type** (Union[[`PromptEncoderReparameterizationType`], `str`]): The encoder type of the prompt + encoder. - Input shape: (batch_size, total_virtual_tokens) + Input shape: (`batch_size`, `total_virtual_tokens`) - Output shape: (batch_size, total_virtual_tokens, token_dim) + Output shape: (`batch_size`, `total_virtual_tokens`, `token_dim`) """ def __init__(self, config): diff --git a/src/peft/tuners/prefix_tuning.py b/src/peft/tuners/prefix_tuning.py index fcb207c..d18000e 100644 --- a/src/peft/tuners/prefix_tuning.py +++ b/src/peft/tuners/prefix_tuning.py @@ -24,7 +24,7 @@ from ..utils import PeftType, PromptLearningConfig @dataclass class PrefixTuningConfig(PromptLearningConfig): """ - This is the configuration class to store the configuration of a [`~peft.PrefixEncoder`]. + This is the configuration class to store the configuration of a [`PrefixEncoder`]. Args: encoder_hidden_size (`int`): The hidden size of the prompt encoder. @@ -48,30 +48,38 @@ class PrefixTuningConfig(PromptLearningConfig): # with some refactor class PrefixEncoder(torch.nn.Module): r""" - The torch.nn model to encode the prefix + The `torch.nn` model to encode the prefix. Args: config ([`PrefixTuningConfig`]): The configuration of the prefix encoder. - Example:: + Example: - >>> from peft import PrefixEncoder, PrefixTuningConfig >>> config = PrefixTuningConfig( - peft_type="PREFIX_TUNING", task_type="SEQ_2_SEQ_LM", num_virtual_tokens=20, token_dim=768, - num_transformer_submodules=1, num_attention_heads=12, num_layers=12, encoder_hidden_size=768 - ) - >>> prefix_encoder = PrefixEncoder(config) + ```py + >>> from peft import PrefixEncoder, PrefixTuningConfig + >>> config = PrefixTuningConfig( + ... peft_type="PREFIX_TUNING", + ... task_type="SEQ_2_SEQ_LM", + ... num_virtual_tokens=20, + ... token_dim=768, + ... num_transformer_submodules=1, + ... num_attention_heads=12, + ... num_layers=12, + ... encoder_hidden_size=768, + ... ) + >>> prefix_encoder = PrefixEncoder(config) + ``` **Attributes**: - - **embedding** (`torch.nn.Embedding`) -- - The embedding layer of the prefix encoder. - - **transform** (`torch.nn.Sequential`) -- The - two-layer MLP to transform the prefix embeddings if `prefix_projection` is `True`. + - **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prefix encoder. + - **transform** (`torch.nn.Sequential`) -- The two-layer MLP to transform the prefix embeddings if + `prefix_projection` is `True`. - **prefix_projection** (`bool`) -- Whether to project the prefix embeddings. - Input shape: (batch_size, num_virtual_tokens) + Input shape: (`batch_size`, `num_virtual_tokens`) - Output shape: (batch_size, num_virtual_tokens, 2*layers*hidden) + Output shape: (`batch_size`, `num_virtual_tokens`, `2*layers*hidden`) """ def __init__(self, config): diff --git a/src/peft/tuners/prompt_tuning.py b/src/peft/tuners/prompt_tuning.py index 1dead1d..6880ff7 100644 --- a/src/peft/tuners/prompt_tuning.py +++ b/src/peft/tuners/prompt_tuning.py @@ -31,14 +31,14 @@ class PromptTuningInit(str, enum.Enum): @dataclass class PromptTuningConfig(PromptLearningConfig): """ - This is the configuration class to store the configuration of a [`~peft.PromptEmbedding`]. + This is the configuration class to store the configuration of a [`PromptEmbedding`]. Args: prompt_tuning_init (Union[[`PromptTuningInit`], `str`]): The initialization of the prompt embedding. - prompt_tuning_init_text ( Optional[`str`]): The text to initialize the prompt embedding. - Only used if `prompt_tuning_init` is `TEXT` - tokenizer_name_or_path ( Optional[`str`]): The name or path of the tokenizer. - Only used if `prompt_tuning_init` is `TEXT` + prompt_tuning_init_text (`str`, *optional*): + The text to initialize the prompt embedding. Only used if `prompt_tuning_init` is `TEXT`. + tokenizer_name_or_path (`str`, *optional*): + The name or path of the tokenizer. Only used if `prompt_tuning_init` is `TEXT`. """ prompt_tuning_init: Union[PromptTuningInit, str] = field( @@ -71,23 +71,33 @@ class PromptEmbedding(torch.nn.Module): word_embeddings (`torch.nn.Module`): The word embeddings of the base transformer model. **Attributes**: - **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prompt embedding. + - **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prompt embedding. - Example:: + Example: - >>> from peft import PromptEmbedding, PromptTuningConfig >>> config = PromptTuningConfig( - peft_type="PROMPT_TUNING", task_type="SEQ_2_SEQ_LM", num_virtual_tokens=20, token_dim=768, - num_transformer_submodules=1, num_attention_heads=12, num_layers=12, prompt_tuning_init="TEXT", - prompt_tuning_init_text="Predict if sentiment of this review is positive, negative or neutral", - tokenizer_name_or_path="t5-base", - ) - >>> # t5_model.shared is the word embeddings of the base model >>> prompt_embedding = PromptEmbedding(config, - t5_model.shared) + ```py + >>> from peft import PromptEmbedding, PromptTuningConfig + >>> config = PromptTuningConfig( + ... peft_type="PROMPT_TUNING", + ... task_type="SEQ_2_SEQ_LM", + ... num_virtual_tokens=20, + ... token_dim=768, + ... num_transformer_submodules=1, + ... num_attention_heads=12, + ... num_layers=12, + ... prompt_tuning_init="TEXT", + ... prompt_tuning_init_text="Predict if sentiment of this review is positive, negative or neutral", + ... tokenizer_name_or_path="t5-base", + ... ) - Input Shape: (batch_size, total_virtual_tokens) + >>> # t5_model.shared is the word embeddings of the base model + >>> prompt_embedding = PromptEmbedding(config, t5_model.shared) + ``` - Output Shape: (batch_size, total_virtual_tokens, token_dim) + Input Shape: (`batch_size`, `total_virtual_tokens`) + + Output Shape: (`batch_size`, `total_virtual_tokens`, `token_dim`) """ def __init__(self, config, word_embeddings): diff --git a/src/peft/utils/config.py b/src/peft/utils/config.py index 2be3817..3ace67a 100644 --- a/src/peft/utils/config.py +++ b/src/peft/utils/config.py @@ -42,7 +42,7 @@ class TaskType(str, enum.Enum): class PeftConfigMixin(PushToHubMixin): r""" This is the base configuration class for PEFT adapter models. It contains all the methods that are common to all - PEFT adapter models. This class inherits from `transformers.utils.PushToHubMixin` which contains the methods to + PEFT adapter models. This class inherits from [`~transformers.utils.PushToHubMixin`] which contains the methods to push your model to the Hub. The method `save_pretrained` will save the configuration of your adapter model in a directory. The method `from_pretrained` will load the configuration of your adapter model from a directory. @@ -65,8 +65,8 @@ class PeftConfigMixin(PushToHubMixin): Args: save_directory (`str`): The directory where the configuration will be saved. - **kwargs: - Additional keyword arguments passed along to the `transformers.utils.PushToHubMixin.push_to_hub` + kwargs: + Additional keyword arguments passed along to the [`~transformers.utils.PushToHubMixin.push_to_hub`] method. """ if os.path.isfile(save_directory): @@ -88,8 +88,8 @@ class PeftConfigMixin(PushToHubMixin): Args: pretrained_model_name_or_path (`str`): - The directory or the hub-id where the configuration is saved. - **kwargs: + The directory or the Hub repository id where the configuration is saved. + kwargs: Additional keyword arguments passed along to the child class initialization. """ if os.path.isfile(os.path.join(pretrained_model_name_or_path, CONFIG_NAME)): @@ -128,7 +128,7 @@ class PeftConfigMixin(PushToHubMixin): @dataclass class PeftConfig(PeftConfigMixin): """ - This is the base configuration class to store the configuration of a :class:`~peft.PeftModel`. + This is the base configuration class to store the configuration of a [`PeftModel`]. Args: peft_type (Union[[`~peft.utils.config.PeftType`], `str`]): The type of Peft method to use. From 8e61e2637020d515f57d3d58ec6f52aba43cfa2a Mon Sep 17 00:00:00 2001 From: Steven Liu Date: Fri, 31 Mar 2023 14:41:14 -0700 Subject: [PATCH 6/6] fix kwargs --- src/peft/peft_model.py | 2 +- src/peft/utils/config.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py index 2b79f4c..0afd047 100644 --- a/src/peft/peft_model.py +++ b/src/peft/peft_model.py @@ -92,7 +92,7 @@ class PeftModel(PushToHubMixin, torch.nn.Module): save_directory (`str`): Directory where the adapter model and configuration files will be saved (will be created if it does not exist). - **kwargs: + kwargs (additional keyword arguments, *optional*): Additional keyword arguments passed along to the `push_to_hub` method. """ if os.path.isfile(save_directory): diff --git a/src/peft/utils/config.py b/src/peft/utils/config.py index 3ace67a..bdd7277 100644 --- a/src/peft/utils/config.py +++ b/src/peft/utils/config.py @@ -65,7 +65,7 @@ class PeftConfigMixin(PushToHubMixin): Args: save_directory (`str`): The directory where the configuration will be saved. - kwargs: + kwargs (additional keyword arguments, *optional*): Additional keyword arguments passed along to the [`~transformers.utils.PushToHubMixin.push_to_hub`] method. """ @@ -89,7 +89,7 @@ class PeftConfigMixin(PushToHubMixin): Args: pretrained_model_name_or_path (`str`): The directory or the Hub repository id where the configuration is saved. - kwargs: + kwargs (additional keyword arguments, *optional*): Additional keyword arguments passed along to the child class initialization. """ if os.path.isfile(os.path.join(pretrained_model_name_or_path, CONFIG_NAME)): @@ -145,8 +145,8 @@ class PeftConfig(PeftConfigMixin): @dataclass class PromptLearningConfig(PeftConfig): """ - This is the base configuration class to store the configuration of a Union[[`~peft.PrefixTuning`], - [`~peft.PromptEncoder`], [`~peft.PromptTuning`]]. + This is the base configuration class to store the configuration of [`PrefixTuning`], [`PromptEncoder`], or + [`PromptTuning`]. Args: num_virtual_tokens (`int`): The number of virtual tokens to use.