This commit is contained in:
Sourab Mangrulkar
2022-12-01 14:47:35 +05:30
parent e3d6568a19
commit 870d360390
2 changed files with 23 additions and 0 deletions
+22
View File
@@ -8,6 +8,28 @@ Supported methods:
3. P-Tuning
4. Prompt Tuning
## Getting started
```python
from transformers import AutoModelForSeq2SeqLM
from pet import get_pet_config,get_pet_model
model_name_or_path = "bigscience/mt0-large"
tokenizer_name_or_path = "bigscience/mt0-large"
config = {
"pet_type":"LORA",
"task_type":"SEQ_2_SEQ_LM",
"r": 8,
"lora_alpha": 32,
"lora_dropout": 0.1
}
pet_config = get_pet_config(config)
model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path)
model = get_pet_model(model, pet_config)
model.print_trainable_parameters()
```
## Models support matrix
### Sequence Classification
+1
View File
@@ -18,6 +18,7 @@ PET_TYPE_TO_CONFIG_MAPPING = {
TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING = {
"t5": ["q", "v"],
"mt5": ["q", "v"],
"bart": ["q_proj", "v_proj"],
"gpt2": ["c_attn"],
"bloom": ["query_key_value"],