wassname/peft
1
0
Fork 0
mirror of https://github.com/wassname/peft.git synced 2026-09-09 11:28:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

clean up docstrings

Browse Source
This commit is contained in:
Steven Liu
2023-03-31 14:30:05 -07:00
parent 7c31f51567
commit 622a5a231e
6 changed files with 235 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/peft/peft_model.py
+127 -67
View File
@@ -45,26 +45,26 @@ from .utils import (
class PeftModel(PushToHubMixin, torch.nn.Module):
"""
Parameter-Efficient Fine-Tuning Model. Base model encompassing various Peft methods.
Base model encompassing various Peft methods.
Args:
model ([`PreTrainedModel`]): The base transformer model used for Peft.
model ([`~transformers.PreTrainedModel`]): The base transformer model used for Peft.
peft_config ([`PeftConfig`]): The configuration of the Peft model.
**Attributes**:
- **base_model** ([`PreTrainedModel`]) -- The base transformer model used for Peft.
- **base_model** ([`~transformers.PreTrainedModel`]) -- The base transformer model used for Peft.
- **peft_config** ([`PeftConfig`]) -- The configuration of the Peft model.
- **modules_to_save** (`list` of `str`) -- The list of sub-module names to save when
saving the model.
- **prompt_encoder** ([`PromptEncoder`]) -- The prompt encoder used for Peft if
`isinstance(self.peft_config, PromptLearningConfig)`.
using [`PromptLearningConfig`].
- **prompt_tokens** (`torch.Tensor`) -- The virtual prompt tokens used for Peft if
`isinstance(self.peft_config, PromptLearningConfig)`.
using [`PromptLearningConfig`].
- **transformer_backbone_name** (`str`) -- The name of the transformer
backbone in the base model if `isinstance(self.peft_config, PromptLearningConfig)`.
backbone in the base model if using [`PromptLearningConfig`].
- **word_embeddings** (`torch.nn.Embedding`) -- The word embeddings of the transformer backbone
in the base model if `isinstance(self.peft_config, PromptLearningConfig)`.
in the base model if using [`PromptLearningConfig`].
"""
def __init__(self, model, peft_config: PeftConfig):
@@ -84,10 +84,11 @@ class PeftModel(PushToHubMixin, torch.nn.Module):
def save_pretrained(self, save_directory, **kwargs):
r"""
Args:
This function saves the adapter model and the adapter configuration files to a directory, so that it can be
re-loaded using the `LoraModel.from_pretrained` class method, and also used by the `LoraModel.push_to_hub`
reloaded using the [`LoraModel.from_pretrained`] class method, and also used by the [`LoraModel.push_to_hub`]
method.
Args:
save_directory (`str`):
Directory where the adapter model and configuration files will be saved (will be created if it does not
exist).
@@ -117,17 +118,18 @@ class PeftModel(PushToHubMixin, torch.nn.Module):
@classmethod
def from_pretrained(cls, model, model_id, **kwargs):
r"""
Instantiate a [`LoraModel`] from a pretrained Lora configuration and weights.
Args:
Instantiate a `LoraModel` from a pretrained Lora configuration and weights.
model (`transformers.PreTrainedModel`):
The model to be adapted. The model should be initialized with the `from_pretrained` method. from
`transformers` library.
model_id (`str`):
model ([`~transformers.PreTrainedModel`]):
The model to be adapted. The model should be initialized with the
[`~transformers.PreTrainedModel.from_pretrained`] method from the 🤗 Transformers library.
model_id (`str` or `os.PathLike`):
The name of the Lora configuration to use. Can be either:
- A string, the `model id` of a Lora configuration hosted inside a model repo on
huggingface Hub
- A path to a directory containing a Lora configuration file saved using the
`save_pretrained` method, e.g., ``./my_lora_config_directory/``.
- A string, the `model id` of a Lora configuration hosted inside a model repo on the Hugging Face
Hub.
- A path to a directory containing a Lora configuration file saved using the `save_pretrained`
method (`./my_lora_config_directory/`).
"""
from .mapping import MODEL_TYPE_TO_PEFT_MODEL_MAPPING, PEFT_TYPE_TO_CONFIG_MAPPING
@@ -322,25 +324,39 @@ class PeftModelForSequenceClassification(PeftModel):
Peft model for sequence classification tasks.
Args:
model ([`PreTrainedModel`]): Base transformer model
model ([`~transformers.PreTrainedModel`]): Base transformer model.
peft_config ([`PeftConfig`]): Peft config.
**Attributes**:
- **config** ([`PretrainedConfig`]) -- The configuration object of the base model.
- **config** ([`~transformers.PretrainedConfig`]) -- The configuration object of the base model.
- **cls_layer_name** (`str`) -- The name of the classification layer.
Example::
Example:
>>> from transformers import AutoModelForSequenceClassification >>> from peft import
PeftModelForSequenceClassification, get_peft_config >>> config = {
'peft_type': 'PREFIX_TUNING', 'task_type': 'SEQ_CLS', 'inference_mode': False, 'num_virtual_tokens':
20, 'token_dim': 768, 'num_transformer_submodules': 1, 'num_attention_heads': 12, 'num_layers': 12,
'encoder_hidden_size': 768, 'prefix_projection': False, 'postprocess_past_key_value_function': None
}
>>> peft_config = get_peft_config(config) >>> model =
AutoModelForSequenceClassification.from_pretrained("bert-base-cased") >>> peft_model =
PeftModelForSequenceClassification(model, peft_config) >>> peft_model.print_trainable_parameters() trainable
params: 370178 || all params: 108680450 || trainable%: 0.3406113979101117
```py
>>> from transformers import AutoModelForSequenceClassification
>>> from peft import PeftModelForSequenceClassification, get_peft_config
>>> config = {
... "peft_type": "PREFIX_TUNING",
... "task_type": "SEQ_CLS",
... "inference_mode": False,
... "num_virtual_tokens": 20,
... "token_dim": 768,
... "num_transformer_submodules": 1,
... "num_attention_heads": 12,
... "num_layers": 12,
... "encoder_hidden_size": 768,
... "prefix_projection": False,
... "postprocess_past_key_value_function": None,
... }
>>> peft_config = get_peft_config(config)
>>> model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased")
>>> peft_model = PeftModelForSequenceClassification(model, peft_config)
>>> peft_model.print_trainable_parameters()
trainable params: 370178 || all params: 108680450 || trainable%: 0.3406113979101117
```
"""
def __init__(self, model, peft_config: PeftConfig):
@@ -490,24 +506,39 @@ class PeftModelForSequenceClassification(PeftModel):
class PeftModelForCausalLM(PeftModel):
"""
Peft model for Causal LM
Peft model for causal language modeling.
Args:
model ([`PreTrainedModel`]): Base transformer model
model ([`~transformers.PreTrainedModel`]): Base transformer model.
peft_config ([`PeftConfig`]): Peft config.
Example::
Example:
```py
>>> from transformers import AutoModelForCausalLM
>>> from peft import PeftModelForCausalLM, get_peft_config
>>> from transformers import AutoModelForCausalLM >>> from peft import PeftModelForCausalLM, get_peft_config
>>> config = {
'peft_type': 'PREFIX_TUNING', 'task_type': 'CAUSAL_LM', 'inference_mode': False, 'num_virtual_tokens':
20, 'token_dim': 1280, 'num_transformer_submodules': 1, 'num_attention_heads': 20, 'num_layers': 36,
'encoder_hidden_size': 1280, 'prefix_projection': False, 'postprocess_past_key_value_function': None
}
>>> peft_config = get_peft_config(config) >>> model = AutoModelForCausalLM.from_pretrained("gpt2-large") >>>
peft_model = PeftModelForCausalLM(model, peft_config) >>> peft_model.print_trainable_parameters() trainable
params: 1843200 || all params: 775873280 || trainable%: 0.23756456724479544
... "peft_type": "PREFIX_TUNING",
... "task_type": "CAUSAL_LM",
... "inference_mode": False,
... "num_virtual_tokens": 20,
... "token_dim": 1280,
... "num_transformer_submodules": 1,
... "num_attention_heads": 20,
... "num_layers": 36,
... "encoder_hidden_size": 1280,
... "prefix_projection": False,
... "postprocess_past_key_value_function": None,
... }
>>> peft_config = get_peft_config(config)
>>> model = AutoModelForCausalLM.from_pretrained("gpt2-large")
>>> peft_model = PeftModelForCausalLM(model, peft_config)
>>> peft_model.print_trainable_parameters()
trainable params: 1843200 || all params: 775873280 || trainable%: 0.23756456724479544
```
"""
def __init__(self, model, peft_config: PeftConfig):
@@ -641,24 +672,39 @@ class PeftModelForCausalLM(PeftModel):
class PeftModelForSeq2SeqLM(PeftModel):
"""
Peft model for Seq2Seq LM
Peft model for sequence-to-sequence language modeling.
Args:
model ([`PreTrainedModel`]): Base transformer model
model ([`~transformers.PreTrainedModel`]): Base transformer model.
peft_config ([`PeftConfig`]): Peft config.
Example::
Example:
```py
>>> from transformers import AutoModelForSeq2SeqLM
>>> from peft import PeftModelForSeq2SeqLM, get_peft_config
>>> from transformers import AutoModelForSeq2SeqLM >>> from peft import PeftModelForSeq2SeqLM, get_peft_config
>>> config = {
'peft_type': 'LORA', 'task_type': 'SEQ_2_SEQ_LM', 'inference_mode': False, 'r': 8, 'target_modules':
['q', 'v'], 'lora_alpha': 32, 'lora_dropout': 0.1, 'merge_weights': False, 'fan_in_fan_out': False,
'enable_lora': None, 'bias': 'none'
}
>>> peft_config = get_peft_config(config) >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") >>>
peft_model = PeftModelForSeq2SeqLM(model, peft_config) >>> peft_model.print_trainable_parameters() trainable
params: 884736 || all params: 223843584 || trainable%: 0.3952474242013566
... "peft_type": "LORA",
... "task_type": "SEQ_2_SEQ_LM",
... "inference_mode": False,
... "r": 8,
... "target_modules": ["q", "v"],
... "lora_alpha": 32,
... "lora_dropout": 0.1,
... "merge_weights": False,
... "fan_in_fan_out": False,
... "enable_lora": None,
... "bias": "none",
... }
>>> peft_config = get_peft_config(config)
>>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base")
>>> peft_model = PeftModelForSeq2SeqLM(model, peft_config)
>>> peft_model.print_trainable_parameters()
trainable params: 884736 || all params: 223843584 || trainable%: 0.3952474242013566
```
"""
def __init__(self, model, peft_config: PeftConfig):
@@ -808,28 +854,42 @@ class PeftModelForSeq2SeqLM(PeftModel):
class PeftModelForTokenClassification(PeftModel):
"""
Peft model for sequence classification tasks.
Peft model for token classification tasks.
Args:
model ([`PreTrainedModel`]): Base transformer model
model ([`~transformers.PreTrainedModel`]): Base transformer model.
peft_config ([`PeftConfig`]): Peft config.
**Attributes**:
- **config** ([`PretrainedConfig`]) -- The configuration object of the base model.
- **config** ([`~transformers.PretrainedConfig`]) -- The configuration object of the base model.
- **cls_layer_name** (`str`) -- The name of the classification layer.
Example::
Example:
>>> from transformers import AutoModelForSequenceClassification >>> from peft import
PeftModelForTokenClassification, get_peft_config >>> config = {
'peft_type': 'PREFIX_TUNING', 'task_type': 'TOKEN_CLS', 'inference_mode': False, 'num_virtual_tokens':
20, 'token_dim': 768, 'num_transformer_submodules': 1, 'num_attention_heads': 12, 'num_layers': 12,
'encoder_hidden_size': 768, 'prefix_projection': False, 'postprocess_past_key_value_function': None
}
>>> peft_config = get_peft_config(config) >>> model =
AutoModelForTokenClassification.from_pretrained("bert-base-cased") >>> peft_model =
PeftModelForTokenClassification(model, peft_config) >>> peft_model.print_trainable_parameters() trainable
params: 370178 || all params: 108680450 || trainable%: 0.3406113979101117
```py
>>> from transformers import AutoModelForSequenceClassification
>>> from peft import PeftModelForTokenClassification, get_peft_config
>>> config = {
... "peft_type": "PREFIX_TUNING",
... "task_type": "TOKEN_CLS",
... "inference_mode": False,
... "num_virtual_tokens": 20,
... "token_dim": 768,
... "num_transformer_submodules": 1,
... "num_attention_heads": 12,
... "num_layers": 12,
... "encoder_hidden_size": 768,
... "prefix_projection": False,
... "postprocess_past_key_value_function": None,
... }
>>> peft_config = get_peft_config(config)
>>> model = AutoModelForTokenClassification.from_pretrained("bert-base-cased")
>>> peft_model = PeftModelForTokenClassification(model, peft_config)
>>> peft_model.print_trainable_parameters()
trainable params: 370178 || all params: 108680450 || trainable%: 0.3406113979101117
```
"""
def __init__(self, model, peft_config: PeftConfig):
src/peft/tuners/lora.py
+25 -14
View File
@@ -39,19 +39,19 @@ if is_bnb_available():
@dataclass
class LoraConfig(PeftConfig):
"""
This is the configuration class to store the configuration of a [`~peft.Lora`].
This is the configuration class to store the configuration of a [`LoraModel`].
Args:
r (`int`): Lora attention dimension
r (`int`): Lora attention dimension.
target_modules (`Union[List[str],str]`): The names of the modules to apply Lora to.
lora_alpha (`float`): The alpha parameter for Lora scaling.
lora_dropout (`float`): The dropout probability for Lora layers.
merge_weights (`bool`):
Whether to merge the weights of the Lora layers with the base transformer model in `eval` mode.
fan_in_fan_out (`bool`): Set this to True if the layer to replace stores weight like (fan_in, fan_out)
enable_lora ( `List[bool]`): Used with `lora.MergedLinear`.
bias (`str`): Bias type for Lora. Can be 'none', 'all' or 'lora_only'
modules_to_save (`List[str]`):List of modules apart from LoRA layers to be set as trainable
fan_in_fan_out (`bool`): Set this to True if the layer to replace stores weight like (`fan_in`, `fan_out`).
enable_lora ( `List[bool]`): Used with [`lora.MergedLinear`].
bias (`str`): Bias type for Lora. Can be `none`, `all` or `lora_only`.
modules_to_save (`List[str]`): List of modules apart from Lora layers to be set as trainable
and saved in the final checkpoint.
"""
@@ -96,22 +96,33 @@ class LoraModel(torch.nn.Module):
Creates Low Rank Adapter (Lora) model from a pretrained transformers model.
Args:
model ([`transformers.PreTrainedModel`]): The model to be adapted.
model ([`~transformers.PreTrainedModel`]): The model to be adapted.
config ([`LoraConfig`]): The configuration of the Lora model.
Returns:
`torch.nn.Module`: The Lora model.
Example::
Example:
>>> from transformers import AutoModelForSeq2SeqLM, LoraConfig >>> from peft import LoraModel, LoraConfig >>>
config = LoraConfig(
peft_type="LORA", task_type="SEQ_2_SEQ_LM", r=8, lora_alpha=32, target_modules=["q", "v"],
lora_dropout=0.01, )
>>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") >>> lora_model = LoraModel(config, model)
```py
>>> from transformers import AutoModelForSeq2SeqLM, LoraConfig
>>> from peft import LoraModel, LoraConfig
>>> config = LoraConfig(
... peft_type="LORA",
... task_type="SEQ_2_SEQ_LM",
... r=8,
... lora_alpha=32,
... target_modules=["q", "v"],
... lora_dropout=0.01,
... )
>>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base")
>>> lora_model = LoraModel(config, model)
```
**Attributes**:
- **model** ([`transformers.PreTrainedModel`]) -- The model to be adapted.
- **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted.
- **peft_config** ([`LoraConfig`]): The configuration of the Lora model.
"""
src/peft/tuners/p_tuning.py
+28 -17
View File
@@ -31,11 +31,11 @@ class PromptEncoderReparameterizationType(str, enum.Enum):
@dataclass
class PromptEncoderConfig(PromptLearningConfig):
"""
This is the configuration class to store the configuration of a [`~peft.PromptEncoder`].
This is the configuration class to store the configuration of a [`PromptEncoder`].
Args:
encoder_reparameterization_type
(Union[[`PromptEncoderReparameterizationType`], `str`]): The type of reparameterization to use.
encoder_reparameterization_type (Union[[`PromptEncoderReparameterizationType`], `str`]):
The type of reparameterization to use.
encoder_hidden_size (`int`): The hidden size of the prompt encoder.
encoder_num_layers (`int`): The number of layers of the prompt encoder.
encoder_dropout (`float`): The dropout probability of the prompt encoder.
@@ -71,19 +71,30 @@ class PromptEncoder(torch.nn.Module):
Args:
config ([`PromptEncoderConfig`]): The configuration of the prompt encoder.
Example::
Example:
>>> from peft import PromptEncoder, PromptEncoderConfig >>> config = PromptEncoderConfig(
peft_type="P_TUNING", task_type="SEQ_2_SEQ_LM", num_virtual_tokens=20, token_dim=768,
num_transformer_submodules=1, num_attention_heads=12, num_layers=12,
encoder_reparameterization_type="MLP", encoder_hidden_size=768
)
>>> prompt_encoder = PromptEncoder(config)
```py
>>> from peft import PromptEncoder, PromptEncoderConfig
>>> config = PromptEncoderConfig(
... peft_type="P_TUNING",
... task_type="SEQ_2_SEQ_LM",
... num_virtual_tokens=20,
... token_dim=768,
... num_transformer_submodules=1,
... num_attention_heads=12,
... num_layers=12,
... encoder_reparameterization_type="MLP",
... encoder_hidden_size=768,
... )
>>> prompt_encoder = PromptEncoder(config)
```
**Attributes**:
- **embedding** ([`~torch.nn.Embedding`]) -- The embedding layer of the prompt encoder.
- **mlp_head** ([`~torch.nn.Sequential`]) -- The MLP head of the prompt encoder if `inference_mode=False`.
- **lstm_head** ([`~torch.nn.LSTM`]) -- The LSTM head of the prompt encoder if `inference_mode=False` and
- **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prompt encoder.
- **mlp_head** (`torch.nn.Sequential`) -- The MLP head of the prompt encoder if `inference_mode=False`.
- **lstm_head** (`torch.nn.LSTM`) -- The LSTM head of the prompt encoder if `inference_mode=False` and
`encoder_reparameterization_type="LSTM"`.
- **token_dim** (`int`) -- The hidden embedding dimension of the base transformer model.
- **input_size** (`int`) -- The input size of the prompt encoder.
@@ -91,13 +102,13 @@ class PromptEncoder(torch.nn.Module):
- **hidden_size** (`int`) -- The hidden size of the prompt encoder.
- **total_virtual_tokens** (`int`): The total number of virtual tokens of the
prompt encoder.
- **encoder_type** (Union[[`PromptEncoderReparameterizationType`], `str`]):
The encoder type of the prompt encoder.
- **encoder_type** (Union[[`PromptEncoderReparameterizationType`], `str`]): The encoder type of the prompt
encoder.
Input shape: (batch_size, total_virtual_tokens)
Input shape: (`batch_size`, `total_virtual_tokens`)
Output shape: (batch_size, total_virtual_tokens, token_dim)
Output shape: (`batch_size`, `total_virtual_tokens`, `token_dim`)
"""
def __init__(self, config):
src/peft/tuners/prefix_tuning.py
+22 -14
View File
@@ -24,7 +24,7 @@ from ..utils import PeftType, PromptLearningConfig
@dataclass
class PrefixTuningConfig(PromptLearningConfig):
"""
This is the configuration class to store the configuration of a [`~peft.PrefixEncoder`].
This is the configuration class to store the configuration of a [`PrefixEncoder`].
Args:
encoder_hidden_size (`int`): The hidden size of the prompt encoder.
@@ -48,30 +48,38 @@ class PrefixTuningConfig(PromptLearningConfig):
# with some refactor
class PrefixEncoder(torch.nn.Module):
r"""
The torch.nn model to encode the prefix
The `torch.nn` model to encode the prefix.
Args:
config ([`PrefixTuningConfig`]): The configuration of the prefix encoder.
Example::
Example:
>>> from peft import PrefixEncoder, PrefixTuningConfig >>> config = PrefixTuningConfig(
peft_type="PREFIX_TUNING", task_type="SEQ_2_SEQ_LM", num_virtual_tokens=20, token_dim=768,
num_transformer_submodules=1, num_attention_heads=12, num_layers=12, encoder_hidden_size=768
)
>>> prefix_encoder = PrefixEncoder(config)
```py
>>> from peft import PrefixEncoder, PrefixTuningConfig
>>> config = PrefixTuningConfig(
... peft_type="PREFIX_TUNING",
... task_type="SEQ_2_SEQ_LM",
... num_virtual_tokens=20,
... token_dim=768,
... num_transformer_submodules=1,
... num_attention_heads=12,
... num_layers=12,
... encoder_hidden_size=768,
... )
>>> prefix_encoder = PrefixEncoder(config)
```
**Attributes**:
- **embedding** (`torch.nn.Embedding`) --
The embedding layer of the prefix encoder.
- **transform** (`torch.nn.Sequential`) -- The
two-layer MLP to transform the prefix embeddings if `prefix_projection` is `True`.
- **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prefix encoder.
- **transform** (`torch.nn.Sequential`) -- The two-layer MLP to transform the prefix embeddings if
`prefix_projection` is `True`.
- **prefix_projection** (`bool`) -- Whether to project the prefix embeddings.
Input shape: (batch_size, num_virtual_tokens)
Input shape: (`batch_size`, `num_virtual_tokens`)
Output shape: (batch_size, num_virtual_tokens, 2*layers*hidden)
Output shape: (`batch_size`, `num_virtual_tokens`, `2*layers*hidden`)
"""
def __init__(self, config):
src/peft/tuners/prompt_tuning.py
+27 -17
View File
@@ -31,14 +31,14 @@ class PromptTuningInit(str, enum.Enum):
@dataclass
class PromptTuningConfig(PromptLearningConfig):
"""
This is the configuration class to store the configuration of a [`~peft.PromptEmbedding`].
This is the configuration class to store the configuration of a [`PromptEmbedding`].
Args:
prompt_tuning_init (Union[[`PromptTuningInit`], `str`]): The initialization of the prompt embedding.
prompt_tuning_init_text ( Optional[`str`]): The text to initialize the prompt embedding.
Only used if `prompt_tuning_init` is `TEXT`
tokenizer_name_or_path ( Optional[`str`]): The name or path of the tokenizer.
Only used if `prompt_tuning_init` is `TEXT`
prompt_tuning_init_text (`str`, *optional*):
The text to initialize the prompt embedding. Only used if `prompt_tuning_init` is `TEXT`.
tokenizer_name_or_path (`str`, *optional*):
The name or path of the tokenizer. Only used if `prompt_tuning_init` is `TEXT`.
"""
prompt_tuning_init: Union[PromptTuningInit, str] = field(
@@ -71,23 +71,33 @@ class PromptEmbedding(torch.nn.Module):
word_embeddings (`torch.nn.Module`): The word embeddings of the base transformer model.
**Attributes**:
**embedding** (`torch.nn.Embedding`) -- The embedding layer of the prompt embedding.
- **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prompt embedding.
Example::
Example:
>>> from peft import PromptEmbedding, PromptTuningConfig >>> config = PromptTuningConfig(
peft_type="PROMPT_TUNING", task_type="SEQ_2_SEQ_LM", num_virtual_tokens=20, token_dim=768,
num_transformer_submodules=1, num_attention_heads=12, num_layers=12, prompt_tuning_init="TEXT",
prompt_tuning_init_text="Predict if sentiment of this review is positive, negative or neutral",
tokenizer_name_or_path="t5-base",
)
>>> # t5_model.shared is the word embeddings of the base model >>> prompt_embedding = PromptEmbedding(config,
t5_model.shared)
```py
>>> from peft import PromptEmbedding, PromptTuningConfig
>>> config = PromptTuningConfig(
... peft_type="PROMPT_TUNING",
... task_type="SEQ_2_SEQ_LM",
... num_virtual_tokens=20,
... token_dim=768,
... num_transformer_submodules=1,
... num_attention_heads=12,
... num_layers=12,
... prompt_tuning_init="TEXT",
... prompt_tuning_init_text="Predict if sentiment of this review is positive, negative or neutral",
... tokenizer_name_or_path="t5-base",
... )
Input Shape: (batch_size, total_virtual_tokens)
>>> # t5_model.shared is the word embeddings of the base model
>>> prompt_embedding = PromptEmbedding(config, t5_model.shared)
```
Output Shape: (batch_size, total_virtual_tokens, token_dim)
Input Shape: (`batch_size`, `total_virtual_tokens`)
Output Shape: (`batch_size`, `total_virtual_tokens`, `token_dim`)
"""
def __init__(self, config, word_embeddings):
src/peft/utils/config.py
+6 -6
View File
@@ -42,7 +42,7 @@ class TaskType(str, enum.Enum):
class PeftConfigMixin(PushToHubMixin):
r"""
This is the base configuration class for PEFT adapter models. It contains all the methods that are common to all
PEFT adapter models. This class inherits from `transformers.utils.PushToHubMixin` which contains the methods to
PEFT adapter models. This class inherits from [`~transformers.utils.PushToHubMixin`] which contains the methods to
push your model to the Hub. The method `save_pretrained` will save the configuration of your adapter model in a
directory. The method `from_pretrained` will load the configuration of your adapter model from a directory.
@@ -65,8 +65,8 @@ class PeftConfigMixin(PushToHubMixin):
Args:
save_directory (`str`):
The directory where the configuration will be saved.
**kwargs:
Additional keyword arguments passed along to the `transformers.utils.PushToHubMixin.push_to_hub`
kwargs:
Additional keyword arguments passed along to the [`~transformers.utils.PushToHubMixin.push_to_hub`]
method.
"""
if os.path.isfile(save_directory):
@@ -88,8 +88,8 @@ class PeftConfigMixin(PushToHubMixin):
Args:
pretrained_model_name_or_path (`str`):
The directory or the hub-id where the configuration is saved.
**kwargs:
The directory or the Hub repository id where the configuration is saved.
kwargs:
Additional keyword arguments passed along to the child class initialization.
"""
if os.path.isfile(os.path.join(pretrained_model_name_or_path, CONFIG_NAME)):
@@ -128,7 +128,7 @@ class PeftConfigMixin(PushToHubMixin):
@dataclass
class PeftConfig(PeftConfigMixin):
"""
This is the base configuration class to store the configuration of a :class:`~peft.PeftModel`.
This is the base configuration class to store the configuration of a [`PeftModel`].
Args:
peft_type (Union[[`~peft.utils.config.PeftType`], `str`]): The type of Peft method to use.