From 622a5a231ef36d2fa114032bcfe130c6299f9a24 Mon Sep 17 00:00:00 2001 From: Steven Liu Date: Fri, 31 Mar 2023 14:30:05 -0700 Subject: [PATCH] clean up docstrings --- src/peft/peft_model.py | 194 ++++++++++++++++++++----------- src/peft/tuners/lora.py | 39 ++++--- src/peft/tuners/p_tuning.py | 45 ++++--- src/peft/tuners/prefix_tuning.py | 36 +++--- src/peft/tuners/prompt_tuning.py | 44 ++++--- src/peft/utils/config.py | 12 +- 6 files changed, 235 insertions(+), 135 deletions(-) diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py index 7491342..2b79f4c 100644 --- a/src/peft/peft_model.py +++ b/src/peft/peft_model.py @@ -45,26 +45,26 @@ from .utils import ( class PeftModel(PushToHubMixin, torch.nn.Module): """ - Parameter-Efficient Fine-Tuning Model. Base model encompassing various Peft methods. + Base model encompassing various Peft methods. Args: - model ([`PreTrainedModel`]): The base transformer model used for Peft. + model ([`~transformers.PreTrainedModel`]): The base transformer model used for Peft. peft_config ([`PeftConfig`]): The configuration of the Peft model. **Attributes**: - - **base_model** ([`PreTrainedModel`]) -- The base transformer model used for Peft. + - **base_model** ([`~transformers.PreTrainedModel`]) -- The base transformer model used for Peft. - **peft_config** ([`PeftConfig`]) -- The configuration of the Peft model. - **modules_to_save** (`list` of `str`) -- The list of sub-module names to save when saving the model. - **prompt_encoder** ([`PromptEncoder`]) -- The prompt encoder used for Peft if - `isinstance(self.peft_config, PromptLearningConfig)`. + using [`PromptLearningConfig`]. - **prompt_tokens** (`torch.Tensor`) -- The virtual prompt tokens used for Peft if - `isinstance(self.peft_config, PromptLearningConfig)`. + using [`PromptLearningConfig`]. - **transformer_backbone_name** (`str`) -- The name of the transformer - backbone in the base model if `isinstance(self.peft_config, PromptLearningConfig)`. + backbone in the base model if using [`PromptLearningConfig`]. - **word_embeddings** (`torch.nn.Embedding`) -- The word embeddings of the transformer backbone - in the base model if `isinstance(self.peft_config, PromptLearningConfig)`. + in the base model if using [`PromptLearningConfig`]. """ def __init__(self, model, peft_config: PeftConfig): @@ -84,10 +84,11 @@ class PeftModel(PushToHubMixin, torch.nn.Module): def save_pretrained(self, save_directory, **kwargs): r""" - Args: This function saves the adapter model and the adapter configuration files to a directory, so that it can be - re-loaded using the `LoraModel.from_pretrained` class method, and also used by the `LoraModel.push_to_hub` + reloaded using the [`LoraModel.from_pretrained`] class method, and also used by the [`LoraModel.push_to_hub`] method. + + Args: save_directory (`str`): Directory where the adapter model and configuration files will be saved (will be created if it does not exist). @@ -117,17 +118,18 @@ class PeftModel(PushToHubMixin, torch.nn.Module): @classmethod def from_pretrained(cls, model, model_id, **kwargs): r""" + Instantiate a [`LoraModel`] from a pretrained Lora configuration and weights. + Args: - Instantiate a `LoraModel` from a pretrained Lora configuration and weights. - model (`transformers.PreTrainedModel`): - The model to be adapted. The model should be initialized with the `from_pretrained` method. from - `transformers` library. - model_id (`str`): + model ([`~transformers.PreTrainedModel`]): + The model to be adapted. The model should be initialized with the + [`~transformers.PreTrainedModel.from_pretrained`] method from the 🤗 Transformers library. + model_id (`str` or `os.PathLike`): The name of the Lora configuration to use. Can be either: - - A string, the `model id` of a Lora configuration hosted inside a model repo on - huggingface Hub - - A path to a directory containing a Lora configuration file saved using the - `save_pretrained` method, e.g., ``./my_lora_config_directory/``. + - A string, the `model id` of a Lora configuration hosted inside a model repo on the Hugging Face + Hub. + - A path to a directory containing a Lora configuration file saved using the `save_pretrained` + method (`./my_lora_config_directory/`). """ from .mapping import MODEL_TYPE_TO_PEFT_MODEL_MAPPING, PEFT_TYPE_TO_CONFIG_MAPPING @@ -322,25 +324,39 @@ class PeftModelForSequenceClassification(PeftModel): Peft model for sequence classification tasks. Args: - model ([`PreTrainedModel`]): Base transformer model + model ([`~transformers.PreTrainedModel`]): Base transformer model. peft_config ([`PeftConfig`]): Peft config. **Attributes**: - - **config** ([`PretrainedConfig`]) -- The configuration object of the base model. + - **config** ([`~transformers.PretrainedConfig`]) -- The configuration object of the base model. - **cls_layer_name** (`str`) -- The name of the classification layer. - Example:: + Example: - >>> from transformers import AutoModelForSequenceClassification >>> from peft import - PeftModelForSequenceClassification, get_peft_config >>> config = { - 'peft_type': 'PREFIX_TUNING', 'task_type': 'SEQ_CLS', 'inference_mode': False, 'num_virtual_tokens': - 20, 'token_dim': 768, 'num_transformer_submodules': 1, 'num_attention_heads': 12, 'num_layers': 12, - 'encoder_hidden_size': 768, 'prefix_projection': False, 'postprocess_past_key_value_function': None - } - >>> peft_config = get_peft_config(config) >>> model = - AutoModelForSequenceClassification.from_pretrained("bert-base-cased") >>> peft_model = - PeftModelForSequenceClassification(model, peft_config) >>> peft_model.print_trainable_parameters() trainable - params: 370178 || all params: 108680450 || trainable%: 0.3406113979101117 + ```py + >>> from transformers import AutoModelForSequenceClassification + >>> from peft import PeftModelForSequenceClassification, get_peft_config + + >>> config = { + ... "peft_type": "PREFIX_TUNING", + ... "task_type": "SEQ_CLS", + ... "inference_mode": False, + ... "num_virtual_tokens": 20, + ... "token_dim": 768, + ... "num_transformer_submodules": 1, + ... "num_attention_heads": 12, + ... "num_layers": 12, + ... "encoder_hidden_size": 768, + ... "prefix_projection": False, + ... "postprocess_past_key_value_function": None, + ... } + + >>> peft_config = get_peft_config(config) + >>> model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased") + >>> peft_model = PeftModelForSequenceClassification(model, peft_config) + >>> peft_model.print_trainable_parameters() + trainable params: 370178 || all params: 108680450 || trainable%: 0.3406113979101117 + ``` """ def __init__(self, model, peft_config: PeftConfig): @@ -490,24 +506,39 @@ class PeftModelForSequenceClassification(PeftModel): class PeftModelForCausalLM(PeftModel): """ - Peft model for Causal LM + Peft model for causal language modeling. Args: - model ([`PreTrainedModel`]): Base transformer model + model ([`~transformers.PreTrainedModel`]): Base transformer model. peft_config ([`PeftConfig`]): Peft config. - Example:: + Example: + + ```py + >>> from transformers import AutoModelForCausalLM + >>> from peft import PeftModelForCausalLM, get_peft_config - >>> from transformers import AutoModelForCausalLM >>> from peft import PeftModelForCausalLM, get_peft_config >>> config = { - 'peft_type': 'PREFIX_TUNING', 'task_type': 'CAUSAL_LM', 'inference_mode': False, 'num_virtual_tokens': - 20, 'token_dim': 1280, 'num_transformer_submodules': 1, 'num_attention_heads': 20, 'num_layers': 36, - 'encoder_hidden_size': 1280, 'prefix_projection': False, 'postprocess_past_key_value_function': None - } - >>> peft_config = get_peft_config(config) >>> model = AutoModelForCausalLM.from_pretrained("gpt2-large") >>> - peft_model = PeftModelForCausalLM(model, peft_config) >>> peft_model.print_trainable_parameters() trainable - params: 1843200 || all params: 775873280 || trainable%: 0.23756456724479544 + ... "peft_type": "PREFIX_TUNING", + ... "task_type": "CAUSAL_LM", + ... "inference_mode": False, + ... "num_virtual_tokens": 20, + ... "token_dim": 1280, + ... "num_transformer_submodules": 1, + ... "num_attention_heads": 20, + ... "num_layers": 36, + ... "encoder_hidden_size": 1280, + ... "prefix_projection": False, + ... "postprocess_past_key_value_function": None, + ... } + + >>> peft_config = get_peft_config(config) + >>> model = AutoModelForCausalLM.from_pretrained("gpt2-large") + >>> peft_model = PeftModelForCausalLM(model, peft_config) + >>> peft_model.print_trainable_parameters() + trainable params: 1843200 || all params: 775873280 || trainable%: 0.23756456724479544 + ``` """ def __init__(self, model, peft_config: PeftConfig): @@ -641,24 +672,39 @@ class PeftModelForCausalLM(PeftModel): class PeftModelForSeq2SeqLM(PeftModel): """ - Peft model for Seq2Seq LM + Peft model for sequence-to-sequence language modeling. Args: - model ([`PreTrainedModel`]): Base transformer model + model ([`~transformers.PreTrainedModel`]): Base transformer model. peft_config ([`PeftConfig`]): Peft config. - Example:: + Example: + + ```py + >>> from transformers import AutoModelForSeq2SeqLM + >>> from peft import PeftModelForSeq2SeqLM, get_peft_config - >>> from transformers import AutoModelForSeq2SeqLM >>> from peft import PeftModelForSeq2SeqLM, get_peft_config >>> config = { - 'peft_type': 'LORA', 'task_type': 'SEQ_2_SEQ_LM', 'inference_mode': False, 'r': 8, 'target_modules': - ['q', 'v'], 'lora_alpha': 32, 'lora_dropout': 0.1, 'merge_weights': False, 'fan_in_fan_out': False, - 'enable_lora': None, 'bias': 'none' - } - >>> peft_config = get_peft_config(config) >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") >>> - peft_model = PeftModelForSeq2SeqLM(model, peft_config) >>> peft_model.print_trainable_parameters() trainable - params: 884736 || all params: 223843584 || trainable%: 0.3952474242013566 + ... "peft_type": "LORA", + ... "task_type": "SEQ_2_SEQ_LM", + ... "inference_mode": False, + ... "r": 8, + ... "target_modules": ["q", "v"], + ... "lora_alpha": 32, + ... "lora_dropout": 0.1, + ... "merge_weights": False, + ... "fan_in_fan_out": False, + ... "enable_lora": None, + ... "bias": "none", + ... } + + >>> peft_config = get_peft_config(config) + >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") + >>> peft_model = PeftModelForSeq2SeqLM(model, peft_config) + >>> peft_model.print_trainable_parameters() + trainable params: 884736 || all params: 223843584 || trainable%: 0.3952474242013566 + ``` """ def __init__(self, model, peft_config: PeftConfig): @@ -808,28 +854,42 @@ class PeftModelForSeq2SeqLM(PeftModel): class PeftModelForTokenClassification(PeftModel): """ - Peft model for sequence classification tasks. + Peft model for token classification tasks. Args: - model ([`PreTrainedModel`]): Base transformer model + model ([`~transformers.PreTrainedModel`]): Base transformer model. peft_config ([`PeftConfig`]): Peft config. **Attributes**: - - **config** ([`PretrainedConfig`]) -- The configuration object of the base model. + - **config** ([`~transformers.PretrainedConfig`]) -- The configuration object of the base model. - **cls_layer_name** (`str`) -- The name of the classification layer. - Example:: + Example: - >>> from transformers import AutoModelForSequenceClassification >>> from peft import - PeftModelForTokenClassification, get_peft_config >>> config = { - 'peft_type': 'PREFIX_TUNING', 'task_type': 'TOKEN_CLS', 'inference_mode': False, 'num_virtual_tokens': - 20, 'token_dim': 768, 'num_transformer_submodules': 1, 'num_attention_heads': 12, 'num_layers': 12, - 'encoder_hidden_size': 768, 'prefix_projection': False, 'postprocess_past_key_value_function': None - } - >>> peft_config = get_peft_config(config) >>> model = - AutoModelForTokenClassification.from_pretrained("bert-base-cased") >>> peft_model = - PeftModelForTokenClassification(model, peft_config) >>> peft_model.print_trainable_parameters() trainable - params: 370178 || all params: 108680450 || trainable%: 0.3406113979101117 + ```py + >>> from transformers import AutoModelForSequenceClassification + >>> from peft import PeftModelForTokenClassification, get_peft_config + + >>> config = { + ... "peft_type": "PREFIX_TUNING", + ... "task_type": "TOKEN_CLS", + ... "inference_mode": False, + ... "num_virtual_tokens": 20, + ... "token_dim": 768, + ... "num_transformer_submodules": 1, + ... "num_attention_heads": 12, + ... "num_layers": 12, + ... "encoder_hidden_size": 768, + ... "prefix_projection": False, + ... "postprocess_past_key_value_function": None, + ... } + + >>> peft_config = get_peft_config(config) + >>> model = AutoModelForTokenClassification.from_pretrained("bert-base-cased") + >>> peft_model = PeftModelForTokenClassification(model, peft_config) + >>> peft_model.print_trainable_parameters() + trainable params: 370178 || all params: 108680450 || trainable%: 0.3406113979101117 + ``` """ def __init__(self, model, peft_config: PeftConfig): diff --git a/src/peft/tuners/lora.py b/src/peft/tuners/lora.py index 47f2c02..51cd56f 100644 --- a/src/peft/tuners/lora.py +++ b/src/peft/tuners/lora.py @@ -39,19 +39,19 @@ if is_bnb_available(): @dataclass class LoraConfig(PeftConfig): """ - This is the configuration class to store the configuration of a [`~peft.Lora`]. + This is the configuration class to store the configuration of a [`LoraModel`]. Args: - r (`int`): Lora attention dimension + r (`int`): Lora attention dimension. target_modules (`Union[List[str],str]`): The names of the modules to apply Lora to. lora_alpha (`float`): The alpha parameter for Lora scaling. lora_dropout (`float`): The dropout probability for Lora layers. merge_weights (`bool`): Whether to merge the weights of the Lora layers with the base transformer model in `eval` mode. - fan_in_fan_out (`bool`): Set this to True if the layer to replace stores weight like (fan_in, fan_out) - enable_lora ( `List[bool]`): Used with `lora.MergedLinear`. - bias (`str`): Bias type for Lora. Can be 'none', 'all' or 'lora_only' - modules_to_save (`List[str]`):List of modules apart from LoRA layers to be set as trainable + fan_in_fan_out (`bool`): Set this to True if the layer to replace stores weight like (`fan_in`, `fan_out`). + enable_lora ( `List[bool]`): Used with [`lora.MergedLinear`]. + bias (`str`): Bias type for Lora. Can be `none`, `all` or `lora_only`. + modules_to_save (`List[str]`): List of modules apart from Lora layers to be set as trainable and saved in the final checkpoint. """ @@ -96,22 +96,33 @@ class LoraModel(torch.nn.Module): Creates Low Rank Adapter (Lora) model from a pretrained transformers model. Args: - model ([`transformers.PreTrainedModel`]): The model to be adapted. + model ([`~transformers.PreTrainedModel`]): The model to be adapted. config ([`LoraConfig`]): The configuration of the Lora model. Returns: `torch.nn.Module`: The Lora model. - Example:: + Example: - >>> from transformers import AutoModelForSeq2SeqLM, LoraConfig >>> from peft import LoraModel, LoraConfig >>> - config = LoraConfig( - peft_type="LORA", task_type="SEQ_2_SEQ_LM", r=8, lora_alpha=32, target_modules=["q", "v"], - lora_dropout=0.01, ) - >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") >>> lora_model = LoraModel(config, model) + ```py + >>> from transformers import AutoModelForSeq2SeqLM, LoraConfig + >>> from peft import LoraModel, LoraConfig + + >>> config = LoraConfig( + ... peft_type="LORA", + ... task_type="SEQ_2_SEQ_LM", + ... r=8, + ... lora_alpha=32, + ... target_modules=["q", "v"], + ... lora_dropout=0.01, + ... ) + + >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") + >>> lora_model = LoraModel(config, model) + ``` **Attributes**: - - **model** ([`transformers.PreTrainedModel`]) -- The model to be adapted. + - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted. - **peft_config** ([`LoraConfig`]): The configuration of the Lora model. """ diff --git a/src/peft/tuners/p_tuning.py b/src/peft/tuners/p_tuning.py index b9c38c4..4a272f3 100644 --- a/src/peft/tuners/p_tuning.py +++ b/src/peft/tuners/p_tuning.py @@ -31,11 +31,11 @@ class PromptEncoderReparameterizationType(str, enum.Enum): @dataclass class PromptEncoderConfig(PromptLearningConfig): """ - This is the configuration class to store the configuration of a [`~peft.PromptEncoder`]. + This is the configuration class to store the configuration of a [`PromptEncoder`]. Args: - encoder_reparameterization_type - (Union[[`PromptEncoderReparameterizationType`], `str`]): The type of reparameterization to use. + encoder_reparameterization_type (Union[[`PromptEncoderReparameterizationType`], `str`]): + The type of reparameterization to use. encoder_hidden_size (`int`): The hidden size of the prompt encoder. encoder_num_layers (`int`): The number of layers of the prompt encoder. encoder_dropout (`float`): The dropout probability of the prompt encoder. @@ -71,19 +71,30 @@ class PromptEncoder(torch.nn.Module): Args: config ([`PromptEncoderConfig`]): The configuration of the prompt encoder. - Example:: + Example: - >>> from peft import PromptEncoder, PromptEncoderConfig >>> config = PromptEncoderConfig( - peft_type="P_TUNING", task_type="SEQ_2_SEQ_LM", num_virtual_tokens=20, token_dim=768, - num_transformer_submodules=1, num_attention_heads=12, num_layers=12, - encoder_reparameterization_type="MLP", encoder_hidden_size=768 - ) - >>> prompt_encoder = PromptEncoder(config) + ```py + >>> from peft import PromptEncoder, PromptEncoderConfig + + >>> config = PromptEncoderConfig( + ... peft_type="P_TUNING", + ... task_type="SEQ_2_SEQ_LM", + ... num_virtual_tokens=20, + ... token_dim=768, + ... num_transformer_submodules=1, + ... num_attention_heads=12, + ... num_layers=12, + ... encoder_reparameterization_type="MLP", + ... encoder_hidden_size=768, + ... ) + + >>> prompt_encoder = PromptEncoder(config) + ``` **Attributes**: - - **embedding** ([`~torch.nn.Embedding`]) -- The embedding layer of the prompt encoder. - - **mlp_head** ([`~torch.nn.Sequential`]) -- The MLP head of the prompt encoder if `inference_mode=False`. - - **lstm_head** ([`~torch.nn.LSTM`]) -- The LSTM head of the prompt encoder if `inference_mode=False` and + - **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prompt encoder. + - **mlp_head** (`torch.nn.Sequential`) -- The MLP head of the prompt encoder if `inference_mode=False`. + - **lstm_head** (`torch.nn.LSTM`) -- The LSTM head of the prompt encoder if `inference_mode=False` and `encoder_reparameterization_type="LSTM"`. - **token_dim** (`int`) -- The hidden embedding dimension of the base transformer model. - **input_size** (`int`) -- The input size of the prompt encoder. @@ -91,13 +102,13 @@ class PromptEncoder(torch.nn.Module): - **hidden_size** (`int`) -- The hidden size of the prompt encoder. - **total_virtual_tokens** (`int`): The total number of virtual tokens of the prompt encoder. - - **encoder_type** (Union[[`PromptEncoderReparameterizationType`], `str`]): - The encoder type of the prompt encoder. + - **encoder_type** (Union[[`PromptEncoderReparameterizationType`], `str`]): The encoder type of the prompt + encoder. - Input shape: (batch_size, total_virtual_tokens) + Input shape: (`batch_size`, `total_virtual_tokens`) - Output shape: (batch_size, total_virtual_tokens, token_dim) + Output shape: (`batch_size`, `total_virtual_tokens`, `token_dim`) """ def __init__(self, config): diff --git a/src/peft/tuners/prefix_tuning.py b/src/peft/tuners/prefix_tuning.py index fcb207c..d18000e 100644 --- a/src/peft/tuners/prefix_tuning.py +++ b/src/peft/tuners/prefix_tuning.py @@ -24,7 +24,7 @@ from ..utils import PeftType, PromptLearningConfig @dataclass class PrefixTuningConfig(PromptLearningConfig): """ - This is the configuration class to store the configuration of a [`~peft.PrefixEncoder`]. + This is the configuration class to store the configuration of a [`PrefixEncoder`]. Args: encoder_hidden_size (`int`): The hidden size of the prompt encoder. @@ -48,30 +48,38 @@ class PrefixTuningConfig(PromptLearningConfig): # with some refactor class PrefixEncoder(torch.nn.Module): r""" - The torch.nn model to encode the prefix + The `torch.nn` model to encode the prefix. Args: config ([`PrefixTuningConfig`]): The configuration of the prefix encoder. - Example:: + Example: - >>> from peft import PrefixEncoder, PrefixTuningConfig >>> config = PrefixTuningConfig( - peft_type="PREFIX_TUNING", task_type="SEQ_2_SEQ_LM", num_virtual_tokens=20, token_dim=768, - num_transformer_submodules=1, num_attention_heads=12, num_layers=12, encoder_hidden_size=768 - ) - >>> prefix_encoder = PrefixEncoder(config) + ```py + >>> from peft import PrefixEncoder, PrefixTuningConfig + >>> config = PrefixTuningConfig( + ... peft_type="PREFIX_TUNING", + ... task_type="SEQ_2_SEQ_LM", + ... num_virtual_tokens=20, + ... token_dim=768, + ... num_transformer_submodules=1, + ... num_attention_heads=12, + ... num_layers=12, + ... encoder_hidden_size=768, + ... ) + >>> prefix_encoder = PrefixEncoder(config) + ``` **Attributes**: - - **embedding** (`torch.nn.Embedding`) -- - The embedding layer of the prefix encoder. - - **transform** (`torch.nn.Sequential`) -- The - two-layer MLP to transform the prefix embeddings if `prefix_projection` is `True`. + - **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prefix encoder. + - **transform** (`torch.nn.Sequential`) -- The two-layer MLP to transform the prefix embeddings if + `prefix_projection` is `True`. - **prefix_projection** (`bool`) -- Whether to project the prefix embeddings. - Input shape: (batch_size, num_virtual_tokens) + Input shape: (`batch_size`, `num_virtual_tokens`) - Output shape: (batch_size, num_virtual_tokens, 2*layers*hidden) + Output shape: (`batch_size`, `num_virtual_tokens`, `2*layers*hidden`) """ def __init__(self, config): diff --git a/src/peft/tuners/prompt_tuning.py b/src/peft/tuners/prompt_tuning.py index 1dead1d..6880ff7 100644 --- a/src/peft/tuners/prompt_tuning.py +++ b/src/peft/tuners/prompt_tuning.py @@ -31,14 +31,14 @@ class PromptTuningInit(str, enum.Enum): @dataclass class PromptTuningConfig(PromptLearningConfig): """ - This is the configuration class to store the configuration of a [`~peft.PromptEmbedding`]. + This is the configuration class to store the configuration of a [`PromptEmbedding`]. Args: prompt_tuning_init (Union[[`PromptTuningInit`], `str`]): The initialization of the prompt embedding. - prompt_tuning_init_text ( Optional[`str`]): The text to initialize the prompt embedding. - Only used if `prompt_tuning_init` is `TEXT` - tokenizer_name_or_path ( Optional[`str`]): The name or path of the tokenizer. - Only used if `prompt_tuning_init` is `TEXT` + prompt_tuning_init_text (`str`, *optional*): + The text to initialize the prompt embedding. Only used if `prompt_tuning_init` is `TEXT`. + tokenizer_name_or_path (`str`, *optional*): + The name or path of the tokenizer. Only used if `prompt_tuning_init` is `TEXT`. """ prompt_tuning_init: Union[PromptTuningInit, str] = field( @@ -71,23 +71,33 @@ class PromptEmbedding(torch.nn.Module): word_embeddings (`torch.nn.Module`): The word embeddings of the base transformer model. **Attributes**: - **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prompt embedding. + - **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prompt embedding. - Example:: + Example: - >>> from peft import PromptEmbedding, PromptTuningConfig >>> config = PromptTuningConfig( - peft_type="PROMPT_TUNING", task_type="SEQ_2_SEQ_LM", num_virtual_tokens=20, token_dim=768, - num_transformer_submodules=1, num_attention_heads=12, num_layers=12, prompt_tuning_init="TEXT", - prompt_tuning_init_text="Predict if sentiment of this review is positive, negative or neutral", - tokenizer_name_or_path="t5-base", - ) - >>> # t5_model.shared is the word embeddings of the base model >>> prompt_embedding = PromptEmbedding(config, - t5_model.shared) + ```py + >>> from peft import PromptEmbedding, PromptTuningConfig + >>> config = PromptTuningConfig( + ... peft_type="PROMPT_TUNING", + ... task_type="SEQ_2_SEQ_LM", + ... num_virtual_tokens=20, + ... token_dim=768, + ... num_transformer_submodules=1, + ... num_attention_heads=12, + ... num_layers=12, + ... prompt_tuning_init="TEXT", + ... prompt_tuning_init_text="Predict if sentiment of this review is positive, negative or neutral", + ... tokenizer_name_or_path="t5-base", + ... ) - Input Shape: (batch_size, total_virtual_tokens) + >>> # t5_model.shared is the word embeddings of the base model + >>> prompt_embedding = PromptEmbedding(config, t5_model.shared) + ``` - Output Shape: (batch_size, total_virtual_tokens, token_dim) + Input Shape: (`batch_size`, `total_virtual_tokens`) + + Output Shape: (`batch_size`, `total_virtual_tokens`, `token_dim`) """ def __init__(self, config, word_embeddings): diff --git a/src/peft/utils/config.py b/src/peft/utils/config.py index 2be3817..3ace67a 100644 --- a/src/peft/utils/config.py +++ b/src/peft/utils/config.py @@ -42,7 +42,7 @@ class TaskType(str, enum.Enum): class PeftConfigMixin(PushToHubMixin): r""" This is the base configuration class for PEFT adapter models. It contains all the methods that are common to all - PEFT adapter models. This class inherits from `transformers.utils.PushToHubMixin` which contains the methods to + PEFT adapter models. This class inherits from [`~transformers.utils.PushToHubMixin`] which contains the methods to push your model to the Hub. The method `save_pretrained` will save the configuration of your adapter model in a directory. The method `from_pretrained` will load the configuration of your adapter model from a directory. @@ -65,8 +65,8 @@ class PeftConfigMixin(PushToHubMixin): Args: save_directory (`str`): The directory where the configuration will be saved. - **kwargs: - Additional keyword arguments passed along to the `transformers.utils.PushToHubMixin.push_to_hub` + kwargs: + Additional keyword arguments passed along to the [`~transformers.utils.PushToHubMixin.push_to_hub`] method. """ if os.path.isfile(save_directory): @@ -88,8 +88,8 @@ class PeftConfigMixin(PushToHubMixin): Args: pretrained_model_name_or_path (`str`): - The directory or the hub-id where the configuration is saved. - **kwargs: + The directory or the Hub repository id where the configuration is saved. + kwargs: Additional keyword arguments passed along to the child class initialization. """ if os.path.isfile(os.path.join(pretrained_model_name_or_path, CONFIG_NAME)): @@ -128,7 +128,7 @@ class PeftConfigMixin(PushToHubMixin): @dataclass class PeftConfig(PeftConfigMixin): """ - This is the base configuration class to store the configuration of a :class:`~peft.PeftModel`. + This is the base configuration class to store the configuration of a [`PeftModel`]. Args: peft_type (Union[[`~peft.utils.config.PeftType`], `str`]): The type of Peft method to use.