From 3d00af47994dbafd171dc007db8928da636a023a Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Fri, 24 Mar 2023 13:16:26 +0530 Subject: [PATCH 01/30] add docs --- .github/workflows/build_documentation.yml | 17 ++ .github/workflows/build_pr_documentation.yml | 16 + .github/workflows/delete_doc_comment.yml | 13 + Makefile | 6 +- docs/Makefile | 19 ++ docs/README.md | 267 +++++++++++++++++ docs/_toctree.yml | 16 + docs/index.mdx | 49 +++ docs/install.mdx | 46 +++ docs/package_reference/config | 0 docs/package_reference/peft_model | 0 docs/package_reference/tuners | 0 docs/quicktour.mdx | 300 +++++++++++++++++++ examples/lora_dreambooth/train_dreambooth.py | 4 +- 14 files changed, 749 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/build_documentation.yml create mode 100644 .github/workflows/build_pr_documentation.yml create mode 100644 .github/workflows/delete_doc_comment.yml create mode 100644 docs/Makefile create mode 100644 docs/README.md create mode 100644 docs/_toctree.yml create mode 100644 docs/index.mdx create mode 100644 docs/install.mdx create mode 100644 docs/package_reference/config create mode 100644 docs/package_reference/peft_model create mode 100644 docs/package_reference/tuners create mode 100644 docs/quicktour.mdx diff --git a/.github/workflows/build_documentation.yml b/.github/workflows/build_documentation.yml new file mode 100644 index 0000000..082ece2 --- /dev/null +++ b/.github/workflows/build_documentation.yml @@ -0,0 +1,17 @@ +name: Build documentation + +on: + push: + branches: + - main + - doc-builder* + - v*-release + +jobs: + build: + uses: huggingface/doc-builder/.github/workflows/build_main_documentation.yml@main + with: + commit_sha: ${{ github.sha }} + package: accelerate + secrets: + token: ${{ secrets.HUGGINGFACE_PUSH }} diff --git a/.github/workflows/build_pr_documentation.yml b/.github/workflows/build_pr_documentation.yml new file mode 100644 index 0000000..7506143 --- /dev/null +++ b/.github/workflows/build_pr_documentation.yml @@ -0,0 +1,16 @@ +name: Build PR Documentation + +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build: + uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@main + with: + commit_sha: ${{ github.event.pull_request.head.sha }} + pr_number: ${{ github.event.number }} + package: peft diff --git a/.github/workflows/delete_doc_comment.yml b/.github/workflows/delete_doc_comment.yml new file mode 100644 index 0000000..e86cc2d --- /dev/null +++ b/.github/workflows/delete_doc_comment.yml @@ -0,0 +1,13 @@ +name: Delete dev documentation + +on: + pull_request: + types: [ closed ] + + +jobs: + delete: + uses: huggingface/doc-builder/.github/workflows/delete_doc_comment.yml@main + with: + pr_number: ${{ github.event.number }} + package: peft diff --git a/Makefile b/Makefile index 61549db..3b6db1f 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: quality style test docs -check_dirs := src tests examples +check_dirs := src tests examples docs # Check that source code meets quality standards @@ -8,13 +8,13 @@ check_dirs := src tests examples quality: black --check $(check_dirs) ruff $(check_dirs) - doc-builder style src tests --max_len 119 --check_only + doc-builder style src tests docs --max_len 119 --check_only # Format source code automatically and check is there are any problems left that need manual fixing style: black $(check_dirs) ruff $(check_dirs) --fix - doc-builder style src tests --max_len 119 + doc-builder style src tests docs --max_len 119 test: pytest tests/ \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..8879933 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,19 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SOURCEDIR = source +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..32e51f1 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,267 @@ + + +# Generating the documentation + +To generate the documentation, you first have to build it. Several packages are necessary to build the doc, +you can install them with the following command, at the root of the code repository: + +```bash +pip install -e ".[docs]" +``` + +Then you need to install our special tool that builds the documentation: + +```bash +pip install git+https://github.com/huggingface/doc-builder +``` + +--- +**NOTE** + +You only need to generate the documentation to inspect it locally (if you're planning changes and want to +check how they look before committing for instance). You don't have to commit the built documentation. + +--- + +## Building the documentation + +Once you have setup the `doc-builder` and additional packages, you can generate the documentation by +typing the following command: + +```bash +doc-builder build accelerate docs/source/ --build_dir ~/tmp/test-build +``` + +You can adapt the `--build_dir` to set any temporary folder that you prefer. This command will create it and generate +the MDX files that will be rendered as the documentation on the main website. You can inspect them in your favorite +Markdown editor. + +## Previewing the documentation + +To preview the docs, first install the `watchdog` module with: + +```bash +pip install watchdog +``` + +Then run the following command: + +```bash +doc-builder preview {package_name} {path_to_docs} +``` + +For example: + +```bash +doc-builder preview transformers docs/source/en/ +``` + +The docs will be viewable at [http://localhost:3000](http://localhost:3000). You can also preview the docs once you have opened a PR. You will see a bot add a comment to a link where the documentation with your changes lives. + +--- +**NOTE** + +The `preview` command only works with existing doc files. When you add a completely new file, you need to update `_toctree.yml` & restart `preview` command (`ctrl-c` to stop it & call `doc-builder preview ...` again). + +--- + +## Adding a new element to the navigation bar + +Accepted files are Markdown (.md or .mdx). + +Create a file with its extension and put it in the source directory. You can then link it to the toc-tree by putting +the filename without the extension in the [`_toctree.yml`](https://github.com/huggingface/accelerate/blob/main/docs/source/_toctree.yml) file. + +## Renaming section headers and moving sections + +It helps to keep the old links working when renaming the section header and/or moving sections from one document to another. This is because the old links are likely to be used in Issues, Forums, and Social media and it'd make for a much more superior user experience if users reading those months later could still easily navigate to the originally intended information. + +Therefore, we simply keep a little map of moved sections at the end of the document where the original section was. The key is to preserve the original anchor. + +So if you renamed a section from: "Section A" to "Section B", then you can add at the end of the file: + +``` +Sections that were moved: + +[ Section A ] +``` +and of course, if you moved it to another file, then: + +``` +Sections that were moved: + +[ Section A ] +``` + +Use the relative style to link to the new file so that the versioned docs continue to work. + + +## Writing Documentation - Specification + +The `huggingface/accelerate` documentation follows the +[Google documentation](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) style for docstrings, +although we can write them directly in Markdown. + +### Adding a new tutorial + +Adding a new tutorial or section is done in two steps: + +- Add a new file under `./source`. This file can either be ReStructuredText (.rst) or Markdown (.md). +- Link that file in `./source/_toctree.yml` on the correct toc-tree. + +Make sure to put your new file under the proper section. It's unlikely to go in the first section (*Get Started*), so +depending on the intended targets (beginners, more advanced users, or researchers) it should go in sections two, three, or +four. + +### Writing source documentation + +Values that should be put in `code` should either be surrounded by backticks: \`like so\`. Note that argument names +and objects like True, None, or any strings should usually be put in `code`. + +When mentioning a class, function, or method, it is recommended to use our syntax for internal links so that our tool +adds a link to its documentation with this syntax: \[\`XXXClass\`\] or \[\`function\`\]. This requires the class or +function to be in the main package. + +If you want to create a link to some internal class or function, you need to +provide its path. For instance: \[\`utils.gather\`\]. This will be converted into a link with +`utils.gather` in the description. To get rid of the path and only keep the name of the object you are +linking to in the description, add a ~: \[\`~utils.gather\`\] will generate a link with `gather` in the description. + +The same works for methods so you can either use \[\`XXXClass.method\`\] or \[~\`XXXClass.method\`\]. + +#### Defining arguments in a method + +Arguments should be defined with the `Args:` (or `Arguments:` or `Parameters:`) prefix, followed by a line return and +an indentation. The argument should be followed by its type, with its shape if it is a tensor, a colon, and its +description: + +``` + Args: + n_layers (`int`): The number of layers of the model. +``` + +If the description is too long to fit in one line (more than 119 characters in total), another indentation is necessary +before writing the description after the argument. + +Finally, to maintain uniformity if any *one* description is too long to fit on one line, the +rest of the parameters should follow suit and have an indention before their description. + +Here's an example showcasing everything so far: + +``` + Args: + gradient_accumulation_steps (`int`, *optional*, default to 1): + The number of steps that should pass before gradients are accumulated. A number > 1 should be combined with `Accelerator.accumulate`. + cpu (`bool`, *optional*): + Whether or not to force the script to execute on CPU. Will ignore GPU available if set to `True` and force the execution on one process only. +``` + +For optional arguments or arguments with defaults we follow the following syntax: imagine we have a function with the +following signature: + +``` +def my_function(x: str = None, a: float = 1): +``` + +then its documentation should look like this: + +``` + Args: + x (`str`, *optional*): + This argument controls ... and has a description longer than 119 chars. + a (`float`, *optional*, defaults to 1): + This argument is used to ... and has a description longer than 119 chars. +``` + +Note that we always omit the "defaults to \`None\`" when None is the default for any argument. Also note that even +if the first line describing your argument type and its default gets long, you can't break it on several lines. You can +however write as many lines as you want in the indented description (see the example above with `input_ids`). + +#### Writing a multi-line code block + +Multi-line code blocks can be useful for displaying examples. They are done between two lines of three backticks as usual in Markdown: + + +```` +```python +# first line of code +# second line +# etc +``` +```` + +#### Writing a return block + +The return block should be introduced with the `Returns:` prefix, followed by a line return and an indentation. +The first line should be the type of the return, followed by a line return. No need to indent further for the elements +building the return. + +Here's an example of a single value return: + +``` + Returns: + `List[int]`: A list of integers in the range [0, 1] --- 1 for a special token, 0 for a sequence token. +``` + +Here's an example of a tuple return, comprising several objects: + +``` + Returns: + `tuple(torch.FloatTensor)` comprising various elements depending on the configuration ([`BertConfig`]) and inputs: + - ** loss** (*optional*, returned when `masked_lm_labels` is provided) `torch.FloatTensor` of shape `(1,)` -- + Total loss is the sum of the masked language modeling loss and the next sequence prediction (classification) loss. + - **prediction_scores** (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`) -- + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). +``` + +## Styling the docstring + +We have an automatic script running with the `make style` comment that will make sure that: +- the docstrings fully take advantage of the line width +- all code examples are formatted using black, like the code of the Transformers library + +This script may have some weird failures if you made a syntax mistake or if you uncover a bug. Therefore, it's +recommended to commit your changes before running `make style`, so you can revert the changes done by that script +easily. + +## Writing documentation examples + +The syntax for Example docstrings can look as follows: + +``` + Example: + + ```python + >>> import time + >>> from accelerate import Accelerator + >>> accelerator = Accelerator() + >>> if accelerator.is_main_process: + ... time.sleep(2) + >>> else: + ... print("I'm waiting for the main process to finish its sleep...") + >>> accelerator.wait_for_everyone() + >>> # Should print on every process at the same time + >>> print("Everyone is here") + ``` +``` + +The docstring should give a minimal, clear example of how the respective function +is to be used in inference and also include the expected (ideally sensible) +output. +Often, readers will try out the example before even going through the function +or class definitions. Therefore, it is of utmost importance that the example +works as expected. \ No newline at end of file diff --git a/docs/_toctree.yml b/docs/_toctree.yml new file mode 100644 index 0000000..4d6dd8b --- /dev/null +++ b/docs/_toctree.yml @@ -0,0 +1,16 @@ +- title: Get Started + sections: + - local: index + title: 🤗 PEFT + - local: quicktour + title: Quicktour + - local: installation + title: Installation +- title: Reference + sections: + - local: package_reference/peft_model + title: PEFT model + - local: package_reference/configs + title: Configuration + - local: package_reference/tuners + title: Tuners \ No newline at end of file diff --git a/docs/index.mdx b/docs/index.mdx new file mode 100644 index 0000000..9e1b4bd --- /dev/null +++ b/docs/index.mdx @@ -0,0 +1,49 @@ + + +# PEFT + +🤗 PEFT is a library that enables using State-of-the-art Parameter-Efficient Fine-Tuning (PEFT) methods. + +PEFT methods enable efficient adaptation of pre-trained language models (PLMs) to +various downstream applications without fine-tuning all the model's parameters. +Fine-tuning large-scale PLMs is often prohibitively costly. +In this regard, PEFT methods only fine-tune a small number of (extra) model parameters, +thereby greatly decreasing the computational and storage costs. +Recent State-of-the-Art PEFT techniques achieve performance comparable to that of full fine-tuning. + +Seamlessly integrated with 🤗 Accelerate for large scale models leveraging DeepSpeed and Big Model Inference. + +Supported methods, with more coming soon: + +1. LoRA: [LORA: LOW-RANK ADAPTATION OF LARGE LANGUAGE MODELS](https://arxiv.org/pdf/2106.09685.pdf) +2. Prefix Tuning: [Prefix-Tuning: Optimizing Continuous Prompts for Generation](https://aclanthology.org/2021.acl-long.353/), [P-Tuning v2: Prompt Tuning Can Be Comparable to Fine-tuning Universally Across Scales and Tasks](https://arxiv.org/pdf/2110.07602.pdf) +3. P-Tuning: [GPT Understands, Too](https://arxiv.org/pdf/2103.10385.pdf) +4. Prompt Tuning: [The Power of Scale for Parameter-Efficient Prompt Tuning](https://arxiv.org/pdf/2104.08691.pdf) + +## Getting started + +```python +from transformers import AutoModelForSeq2SeqLM +from peft import get_peft_config, get_peft_model, LoraConfig, TaskType + +model_name_or_path = "bigscience/mt0-large" +tokenizer_name_or_path = "bigscience/mt0-large" + +peft_config = LoraConfig(task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1) + +model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path) +model = get_peft_model(model, peft_config) +model.print_trainable_parameters() +# output: trainable params: 2359296 || all params: 1231940608 || trainable%: 0.19151053100118282 +``` + diff --git a/docs/install.mdx b/docs/install.mdx new file mode 100644 index 0000000..e086ed8 --- /dev/null +++ b/docs/install.mdx @@ -0,0 +1,46 @@ + + +# Installation and Configuration + +Before you start, you will need to setup your environment, install the appropriate packages, and configure 🤗 PEFT. 🤗 PEFT is tested on **Python 3.7+**. + +## Installing 🤗 PEFT + +🤗 PEFT is available on pypi, as well as on GitHub. Details to install from each are below: + +### pip + +To install 🤗 PEFT from pypi, perform: + +```bash +pip install peft +``` + +### Source + +New features are added every day that haven't been released yet. To try them out yourself, install +from the GitHub repository: + +```bash +pip install git+https://github.com/huggingface/peft +``` + +If you're working on contributing to the library or wish to play with the source code and see live +results as you run the code, an editable version can be installed from a locally-cloned version of the +repository: + +```bash +git clone https://github.com/huggingface/peft +cd peft +pip install -e . +``` diff --git a/docs/package_reference/config b/docs/package_reference/config new file mode 100644 index 0000000..e69de29 diff --git a/docs/package_reference/peft_model b/docs/package_reference/peft_model new file mode 100644 index 0000000..e69de29 diff --git a/docs/package_reference/tuners b/docs/package_reference/tuners new file mode 100644 index 0000000..e69de29 diff --git a/docs/quicktour.mdx b/docs/quicktour.mdx new file mode 100644 index 0000000..45ee22c --- /dev/null +++ b/docs/quicktour.mdx @@ -0,0 +1,300 @@ + + +# Quick tour + +Let's have a look at the 🤗 PEFT main features and traps to avoid. + +## Main use + +To use 🤗 PEFT in your script, you have to follow below steps: + +1. Create a `PeftConfig` object corresponding to your PEFT method. +Please refer to the [Config Page](package_reference/config) for more details. +Below, we will use `LoRAConfig` for demonstration. + +```python +from peft import LoraConfig, TaskType + +peft_config = LoraConfig(task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1) +``` + +Here, `task_type` is the type of task you are training your model for. +For available task types, please refer [TaskType](package_reference/config#peft.config.TaskType). + +2. Load the base model you want to fine-tune. + +```python +from transformers import AutoModelForSeq2SeqLM + +model_name_or_path = "bigscience/mt0-large" +tokenizer_name_or_path = "bigscience/mt0-large" +model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path) +``` + +3. Preprocess your model if you use `bitsandbytes` for INT-8 quantized training; else skip this step. + +```python +from peft import prepare_model_for_int8_training + +model = prepare_model_for_int8_training(model) +``` + +4. Wrap your model in the `PeftModel` object using the `get_peft_model` function. Also, check the number of trainable parameters of your model. + +```python +from peft import get_peft_model + +model = get_peft_model(model, peft_config) +model.print_trainable_parameters() +# output: trainable params: 2359296 || all params: 1231940608 || trainable%: 0.19151053100118282 +``` + +5. Voila 🎉. Now, train the model using 🤗 Transformers Trainer API, 🤗 Accelerate or any custom PyTroch training loop. +Please refer example [peft_lora_seq2seq.ipynb](https://github.com/huggingface/peft/blob/main/examples/conditional_generation/peft_lora_seq2seq.ipynb) for an end-to-end example. + +### Saving/loading a model + +1. Save your model using the `save_pretrained` function. + +```python +model.save_pretrained("output_dir") +# model.push_to_hub("my_awesome_peft_model") also works +``` + +This will only save the incremental PEFT weights that were trained. +For example, you can find the `bigscience/T0_3B` tuned using LoRA on the `twitter_complaints` raft dataset here: +[smangrul/twitter_complaints_bigscience_T0_3B_LORA_SEQ_2_SEQ_LM](https://huggingface.co/smangrul/twitter_complaints_bigscience_T0_3B_LORA_SEQ_2_SEQ_LM). +Notice that it only contains 2 files: `adapter_config.json` and `adapter_model.bin` with the latter being just 19MB. + +2. Load your model using the `from_pretrained` function. + +```diff + from transformers import AutoModelForSeq2SeqLM ++ from peft import PeftModel, PeftConfig + ++ peft_model_id = "smangrul/twitter_complaints_bigscience_T0_3B_LORA_SEQ_2_SEQ_LM" ++ config = PeftConfig.from_pretrained(peft_model_id) + model = AutoModelForSeq2SeqLM.from_pretrained(config.base_model_name_or_path) ++ model = PeftModel.from_pretrained(model, peft_model_id) + tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path) + + model = model.to(device) + model.eval() + inputs = tokenizer("Tweet text : @HondaCustSvc Your customer service has been horrible during the recall process. I will never purchase a Honda again. Label :", return_tensors="pt") + + with torch.no_grad(): + outputs = model.generate(input_ids=inputs["input_ids"].to("cuda"), max_new_tokens=10) + print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True)[0]) +# 'complaint' +``` + +## Launching your distributed script + +PEFT models work with 🤗 Accelerate out of the box. +Use 🤗 Accelerate for Distributed training on various hardware such as GPUs, Apple Silicon devices etc during training. +Use 🤗 Accelerate for inferencing on consumer hardware with small resources. + +### Example of PEFT model training using 🤗 Accelerate's DeepSpeed integration + +DeepSpeed version required `v0.8.0`. An example is provided in `~examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py`. + a. First, run `accelerate config --config_file ds_zero3_cpu.yaml` and answer the questionnaire. + Below are the contents of the config file. + ```yaml + compute_environment: LOCAL_MACHINE + deepspeed_config: + gradient_accumulation_steps: 1 + gradient_clipping: 1.0 + offload_optimizer_device: cpu + offload_param_device: cpu + zero3_init_flag: true + zero3_save_16bit_model: true + zero_stage: 3 + distributed_type: DEEPSPEED + downcast_bf16: 'no' + dynamo_backend: 'NO' + fsdp_config: {} + machine_rank: 0 + main_training_function: main + megatron_lm_config: {} + mixed_precision: 'no' + num_machines: 1 + num_processes: 1 + rdzv_backend: static + same_network: true + use_cpu: false + ``` + b. run the below command to launch the example script + ```bash + accelerate launch --config_file ds_zero3_cpu.yaml examples/peft_lora_seq2seq_accelerate_ds_zero3_offload.py + ``` + + c. output logs: + ```bash + GPU Memory before entering the train : 1916 + GPU Memory consumed at the end of the train (end-begin): 66 + GPU Peak Memory consumed during the train (max-begin): 7488 + GPU Total Peak Memory consumed during the train (max): 9404 + CPU Memory before entering the train : 19411 + CPU Memory consumed at the end of the train (end-begin): 0 + CPU Peak Memory consumed during the train (max-begin): 0 + CPU Total Peak Memory consumed during the train (max): 19411 + epoch=4: train_ppl=tensor(1.0705, device='cuda:0') train_epoch_loss=tensor(0.0681, device='cuda:0') + 100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:27<00:00, 3.92s/it] + GPU Memory before entering the eval : 1982 + GPU Memory consumed at the end of the eval (end-begin): -66 + GPU Peak Memory consumed during the eval (max-begin): 672 + GPU Total Peak Memory consumed during the eval (max): 2654 + CPU Memory before entering the eval : 19411 + CPU Memory consumed at the end of the eval (end-begin): 0 + CPU Peak Memory consumed during the eval (max-begin): 0 + CPU Total Peak Memory consumed during the eval (max): 19411 + accuracy=100.0 + eval_preds[:10]=['no complaint', 'no complaint', 'complaint', 'complaint', 'no complaint', 'no complaint', 'no complaint', 'complaint', 'complaint', 'no complaint'] + dataset['train'][label_column][:10]=['no complaint', 'no complaint', 'complaint', 'complaint', 'no complaint', 'no complaint', 'no complaint', 'complaint', 'complaint', 'no complaint'] + ``` + +### Example of PEFT model inference using 🤗 Accelerate's Big Model Inferencing capabilities +An example is provided in `~examples/causal_language_modeling/peft_lora_clm_accelerate_big_model_inference.ipynb`. + +## Model Support matrix + +### Causal Language Modeling +| Model | LoRA | Prefix Tuning | P-Tuning | Prompt Tuning | +|--------------| ---- | ---- | ---- | ---- | +| GPT-2 | ✅ | ✅ | ✅ | ✅ | +| Bloom | ✅ | ✅ | ✅ | ✅ | +| OPT | ✅ | ✅ | ✅ | ✅ | +| GPT-Neo | ✅ | ✅ | ✅ | ✅ | +| GPT-J | ✅ | ✅ | ✅ | ✅ | +| GPT-NeoX-20B | ✅ | ✅ | ✅ | ✅ | +| LLaMA | ✅ | ✅ | ✅ | ✅ | +| ChatGLM | ✅ | ✅ | ✅ | ✅ | + +### Conditional Generation +| Model | LoRA | Prefix Tuning | P-Tuning | Prompt Tuning | +| --------- | ---- | ---- | ---- | ---- | +| T5 | ✅ | ✅ | ✅ | ✅ | +| BART | ✅ | ✅ | ✅ | ✅ | + +### Sequence Classification +| Model | LoRA | Prefix Tuning | P-Tuning | Prompt Tuning | +| --------- | ---- | ---- | ---- | ---- | +| BERT | ✅ | ✅ | ✅ | ✅ | +| RoBERTa | ✅ | ✅ | ✅ | ✅ | +| GPT-2 | ✅ | ✅ | ✅ | ✅ | +| Bloom | ✅ | ✅ | ✅ | ✅ | +| OPT | ✅ | ✅ | ✅ | ✅ | +| GPT-Neo | ✅ | ✅ | ✅ | ✅ | +| GPT-J | ✅ | ✅ | ✅ | ✅ | +| Deberta | ✅ | | ✅ | ✅ | +| Deberta-v2 | ✅ | | ✅ | ✅ | + +### Token Classification +| Model | LoRA | Prefix Tuning | P-Tuning | Prompt Tuning | +| --------- | ---- | ---- | ---- | ---- | +| BERT | ✅ | ✅ | | | +| RoBERTa | ✅ | ✅ | | | +| GPT-2 | ✅ | ✅ | | | +| Bloom | ✅ | ✅ | | | +| OPT | ✅ | ✅ | | | +| GPT-Neo | ✅ | ✅ | | | +| GPT-J | ✅ | ✅ | | | +| Deberta | ✅ | | | | +| Deberta-v2 | ✅ | | | | + +### Text-to-Image Generation + +| Model | LoRA | Prefix Tuning | P-Tuning | Prompt Tuning | +| --------- | ---- | ---- | ---- | ---- | +| Stable Diffusion | ✅ | | | | + + +### Image Classification + +| Model | LoRA | Prefix Tuning | P-Tuning | Prompt Tuning | +| --------- | ---- | ---- | ---- | ---- | +| ViT | ✅ | | | | +| Swin | ✅ | | | | + +___Note that we have tested LoRA for [ViT](https://huggingface.co/docs/transformers/model_doc/vit) and [Swin](https://huggingface.co/docs/transformers/model_doc/swin) for fine-tuning on image classification. However, it should be possible to use LoRA for any compatible model [provided](https://huggingface.co/models?pipeline_tag=image-classification&sort=downloads&search=vit) by 🤗 Transformers. Check out the respective +examples to learn more. If you run into problems, please open an issue.___ + +The same principle applies to our [segmentation models](https://huggingface.co/models?pipeline_tag=image-segmentation&sort=downloads) as well. + +### Semantic Segmentation + +| Model | LoRA | Prefix Tuning | P-Tuning | Prompt Tuning | +| --------- | ---- | ---- | ---- | ---- | +| SegFormer | ✅ | | | | + + +## Other caveats + +1. Below is an example of using PyTorch FSDP for training. However, it doesn't lead to +any GPU memory savings. Please refer to issue [[FSDP] FSDP with CPU offload consumes 1.65X more GPU memory when training models with most of the params frozen](https://github.com/pytorch/pytorch/issues/91165). + + ```python + from peft.utils.other import fsdp_auto_wrap_policy + + + if os.environ.get("ACCELERATE_USE_FSDP", None) is not None: + accelerator.state.fsdp_plugin.auto_wrap_policy = fsdp_auto_wrap_policy(model) + + model = accelerator.prepare(model) + ``` + + Example of parameter efficient tuning with [`mt0-xxl`](https://huggingface.co/bigscience/mt0-xxl) base model using 🤗 Accelerate is provided in `~examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py`. + a. First, run `accelerate config --config_file fsdp_config.yaml` and answer the questionnaire. + Below are the contents of the config file. + ```yaml + command_file: null + commands: null + compute_environment: LOCAL_MACHINE + deepspeed_config: {} + distributed_type: FSDP + downcast_bf16: 'no' + dynamo_backend: 'NO' + fsdp_config: + fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP + fsdp_backward_prefetch_policy: BACKWARD_PRE + fsdp_offload_params: true + fsdp_sharding_strategy: 1 + fsdp_state_dict_type: FULL_STATE_DICT + fsdp_transformer_layer_cls_to_wrap: T5Block + gpu_ids: null + machine_rank: 0 + main_process_ip: null + main_process_port: null + main_training_function: main + megatron_lm_config: {} + mixed_precision: 'no' + num_machines: 1 + num_processes: 2 + rdzv_backend: static + same_network: true + tpu_name: null + tpu_zone: null + use_cpu: false + ``` + b. run the below command to launch the example script + ```bash + accelerate launch --config_file fsdp_config.yaml examples/peft_lora_seq2seq_accelerate_fsdp.py + ``` + +2. When using `P_TUNING` or `PROMPT_TUNING` with `SEQ_2_SEQ` task, remember to remove the `num_virtual_token` virtual prompt predictions from the left side of the model outputs during evaluations. + +3. For encoder-decoder models, `P_TUNING` or `PROMPT_TUNING` doesn't support the `generate` functionality of transformers because `generate` strictly requires `decoder_input_ids` but +`P_TUNING`/`PROMPT_TUNING` append soft prompt embeddings to `input_embeds` to create +new `input_embeds` to be given to the model. Therefore, `generate` doesn't support this yet. + +4. When using ZeRO3 with zero3_init_flag=True, if you find the GPU memory increase with training steps. we might need to set zero3_init_flag=false in accelerate config.yaml. The related issue is [[BUG] memory leak under zero.Init](https://github.com/microsoft/DeepSpeed/issues/2637) \ No newline at end of file diff --git a/examples/lora_dreambooth/train_dreambooth.py b/examples/lora_dreambooth/train_dreambooth.py index 9145eca..32f78a8 100644 --- a/examples/lora_dreambooth/train_dreambooth.py +++ b/examples/lora_dreambooth/train_dreambooth.py @@ -1063,7 +1063,9 @@ def main(args): ) text_encoder_state_dict = {f"text_encoder_{k}": v for k, v in text_encoder_state_dict.items()} state_dict.update(text_encoder_state_dict) - lora_config["text_encoder_peft_config"] = unwarpped_text_encoder.get_peft_config_as_dict(inference=True) + lora_config["text_encoder_peft_config"] = unwarpped_text_encoder.get_peft_config_as_dict( + inference=True + ) accelerator.print(state_dict) accelerator.save(state_dict, os.path.join(args.output_dir, f"{args.instance_prompt}_lora.pt")) From 13476a807ccd86189809dd00e627da93dfab5aff Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Mon, 27 Mar 2023 13:44:00 +0530 Subject: [PATCH 02/30] Apply suggestions from code review Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- .github/workflows/build_documentation.yml | 2 +- Makefile | 4 +-- docs/README.md | 8 ++--- docs/_toctree.yml | 2 +- docs/index.mdx | 15 +++------ docs/install.mdx | 15 ++++----- docs/quicktour.mdx | 41 +++++++++++------------ 7 files changed, 39 insertions(+), 48 deletions(-) diff --git a/.github/workflows/build_documentation.yml b/.github/workflows/build_documentation.yml index 082ece2..309d35a 100644 --- a/.github/workflows/build_documentation.yml +++ b/.github/workflows/build_documentation.yml @@ -12,6 +12,6 @@ jobs: uses: huggingface/doc-builder/.github/workflows/build_main_documentation.yml@main with: commit_sha: ${{ github.sha }} - package: accelerate + package: peft secrets: token: ${{ secrets.HUGGINGFACE_PUSH }} diff --git a/Makefile b/Makefile index 3b6db1f..145a375 100644 --- a/Makefile +++ b/Makefile @@ -8,13 +8,13 @@ check_dirs := src tests examples docs quality: black --check $(check_dirs) ruff $(check_dirs) - doc-builder style src tests docs --max_len 119 --check_only + doc-builder style src/peft tests docs/source --max_len 119 --check_only # Format source code automatically and check is there are any problems left that need manual fixing style: black $(check_dirs) ruff $(check_dirs) --fix - doc-builder style src tests docs --max_len 119 + doc-builder style src/peft tests docs/source --max_len 119 test: pytest tests/ \ No newline at end of file diff --git a/docs/README.md b/docs/README.md index 32e51f1..5955736 100644 --- a/docs/README.md +++ b/docs/README.md @@ -43,7 +43,7 @@ Once you have setup the `doc-builder` and additional packages, you can generate typing the following command: ```bash -doc-builder build accelerate docs/source/ --build_dir ~/tmp/test-build +doc-builder build peft docs/source/ --build_dir ~/tmp/test-build ``` You can adapt the `--build_dir` to set any temporary folder that you prefer. This command will create it and generate @@ -67,7 +67,7 @@ doc-builder preview {package_name} {path_to_docs} For example: ```bash -doc-builder preview transformers docs/source/en/ +doc-builder preview peft docs/source ``` The docs will be viewable at [http://localhost:3000](http://localhost:3000). You can also preview the docs once you have opened a PR. You will see a bot add a comment to a link where the documentation with your changes lives. @@ -84,7 +84,7 @@ The `preview` command only works with existing doc files. When you add a complet Accepted files are Markdown (.md or .mdx). Create a file with its extension and put it in the source directory. You can then link it to the toc-tree by putting -the filename without the extension in the [`_toctree.yml`](https://github.com/huggingface/accelerate/blob/main/docs/source/_toctree.yml) file. +the filename without the extension in the [`_toctree.yml`](https://github.com/huggingface/peft/blob/main/docs/source/_toctree.yml) file. ## Renaming section headers and moving sections @@ -112,7 +112,7 @@ Use the relative style to link to the new file so that the versioned docs contin ## Writing Documentation - Specification -The `huggingface/accelerate` documentation follows the +The `huggingface/peft` documentation follows the [Google documentation](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) style for docstrings, although we can write them directly in Markdown. diff --git a/docs/_toctree.yml b/docs/_toctree.yml index 4d6dd8b..211b83f 100644 --- a/docs/_toctree.yml +++ b/docs/_toctree.yml @@ -1,7 +1,7 @@ - title: Get Started sections: - local: index - title: 🤗 PEFT + title: 🤗 PEFT - local: quicktour title: Quicktour - local: installation diff --git a/docs/index.mdx b/docs/index.mdx index 9e1b4bd..4f5776f 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -12,18 +12,13 @@ specific language governing permissions and limitations under the License. # PEFT -🤗 PEFT is a library that enables using State-of-the-art Parameter-Efficient Fine-Tuning (PEFT) methods. +🤗 PEFT, or Parameter-Efficient Fine-Tuning (PEFT), is a library for efficiently adapting pre-trained language models (PLMs) to various downstream applications without fine-tuning all the model's parameters. +PEFT methods only fine-tune a small number of (extra) model parameters, significantly decreasing computational and storage costs because fine-tuning large-scale PLMs is prohibitively costly. +Recent state-of-the-art PEFT techniques achieve performance comparable to that of full fine-tuning. -PEFT methods enable efficient adaptation of pre-trained language models (PLMs) to -various downstream applications without fine-tuning all the model's parameters. -Fine-tuning large-scale PLMs is often prohibitively costly. -In this regard, PEFT methods only fine-tune a small number of (extra) model parameters, -thereby greatly decreasing the computational and storage costs. -Recent State-of-the-Art PEFT techniques achieve performance comparable to that of full fine-tuning. +PEFT is seamlessly integrated with 🤗 Accelerate for large-scale models leveraging DeepSpeed and [Big Model Inference](https://huggingface.co/docs/accelerate/usage_guides/big_modeling). -Seamlessly integrated with 🤗 Accelerate for large scale models leveraging DeepSpeed and Big Model Inference. - -Supported methods, with more coming soon: +Supported methods include: 1. LoRA: [LORA: LOW-RANK ADAPTATION OF LARGE LANGUAGE MODELS](https://arxiv.org/pdf/2106.09685.pdf) 2. Prefix Tuning: [Prefix-Tuning: Optimizing Continuous Prompts for Generation](https://aclanthology.org/2021.acl-long.353/), [P-Tuning v2: Prompt Tuning Can Be Comparable to Fine-tuning Universally Across Scales and Tasks](https://arxiv.org/pdf/2110.07602.pdf) diff --git a/docs/install.mdx b/docs/install.mdx index e086ed8..5f5ecff 100644 --- a/docs/install.mdx +++ b/docs/install.mdx @@ -10,26 +10,23 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express o specific language governing permissions and limitations under the License. --> -# Installation and Configuration +# Installation Before you start, you will need to setup your environment, install the appropriate packages, and configure 🤗 PEFT. 🤗 PEFT is tested on **Python 3.7+**. -## Installing 🤗 PEFT +🤗 PEFT is available on pypi, as well as GitHub: -🤗 PEFT is available on pypi, as well as on GitHub. Details to install from each are below: +## pip -### pip - -To install 🤗 PEFT from pypi, perform: +To install 🤗 PEFT from pypi: ```bash pip install peft ``` -### Source +## Source -New features are added every day that haven't been released yet. To try them out yourself, install -from the GitHub repository: +New features that haven't been released yet are added every day, which also means there may be some bugs. To try them out, install from the GitHub repository: ```bash pip install git+https://github.com/huggingface/peft diff --git a/docs/quicktour.mdx b/docs/quicktour.mdx index 45ee22c..e0eb37f 100644 --- a/docs/quicktour.mdx +++ b/docs/quicktour.mdx @@ -12,15 +12,16 @@ specific language governing permissions and limitations under the License. # Quick tour -Let's have a look at the 🤗 PEFT main features and traps to avoid. +Let's have a look at 🤗 PEFT's main features and learn how to set up a `PeftModel` and train it with 🤗 Accelerate's DeepSpeed integration and use it for inference. ## Main use -To use 🤗 PEFT in your script, you have to follow below steps: +To use 🤗 PEFT in your script: -1. Create a `PeftConfig` object corresponding to your PEFT method. -Please refer to the [Config Page](package_reference/config) for more details. -Below, we will use `LoRAConfig` for demonstration. +1. Each PEFT method is defined by a `PeftConfig` object. + +Create a `PeftConfig` object corresponding to your PEFT method (see the [Configuration](package_reference/config) reference for more details) and [`TaskType`], the type of task you're training your model for. +This example trains the [`bigscience/mt0-large`](https://huggingface.co/bigscience/mt0-large) model with the Low-Rank Adaptation of Large Language Models (LoRA) method. Load the `LoRAConfig`, and specify the `task_type` for sequence-to-sequence language modeling. ```python from peft import LoraConfig, TaskType @@ -41,7 +42,7 @@ tokenizer_name_or_path = "bigscience/mt0-large" model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path) ``` -3. Preprocess your model if you use `bitsandbytes` for INT-8 quantized training; else skip this step. +3. Preprocess your model if you use [`bitsandbytes`](https://github.com/TimDettmers/bitsandbytes) for `int8` quantized training; otherwise, skip this step. ```python from peft import prepare_model_for_int8_training @@ -59,8 +60,7 @@ model.print_trainable_parameters() # output: trainable params: 2359296 || all params: 1231940608 || trainable%: 0.19151053100118282 ``` -5. Voila 🎉. Now, train the model using 🤗 Transformers Trainer API, 🤗 Accelerate or any custom PyTroch training loop. -Please refer example [peft_lora_seq2seq.ipynb](https://github.com/huggingface/peft/blob/main/examples/conditional_generation/peft_lora_seq2seq.ipynb) for an end-to-end example. +5. Voila 🎉! Now, train the model using the 🤗 Transformers Trainer API, 🤗 Accelerate, or any custom PyTroch training loop (take a look at the end-to-end [example](https://github.com/huggingface/peft/blob/main/examples/conditional_generation/peft_lora_seq2seq.ipynb) of training [`bigscience/mt0-large`](https://huggingface.co/bigscience/mt0-large)). ### Saving/loading a model @@ -71,10 +71,9 @@ model.save_pretrained("output_dir") # model.push_to_hub("my_awesome_peft_model") also works ``` -This will only save the incremental PEFT weights that were trained. -For example, you can find the `bigscience/T0_3B` tuned using LoRA on the `twitter_complaints` raft dataset here: -[smangrul/twitter_complaints_bigscience_T0_3B_LORA_SEQ_2_SEQ_LM](https://huggingface.co/smangrul/twitter_complaints_bigscience_T0_3B_LORA_SEQ_2_SEQ_LM). -Notice that it only contains 2 files: `adapter_config.json` and `adapter_model.bin` with the latter being just 19MB. +This only saves the incremental PEFT weights that were trained. +For example, [smangrul/twitter_complaints_bigscience_T0_3B_LORA_SEQ_2_SEQ_LM](https://huggingface.co/smangrul/twitter_complaints_bigscience_T0_3B_LORA_SEQ_2_SEQ_LM) is a `bigscience/T0_3B`model finetuned with LoRA on the [`twitter_complaints`](https://huggingface.co/datasets/ought/raft/viewer/twitter_complaints/train) RAFT dataset. +Notice that it only contains 2 files: `adapter_config.json` and `adapter_model.bin`, with the latter being just 19MB. 2. Load your model using the `from_pretrained` function. @@ -101,14 +100,14 @@ Notice that it only contains 2 files: `adapter_config.json` and `adapter_model.b ## Launching your distributed script PEFT models work with 🤗 Accelerate out of the box. -Use 🤗 Accelerate for Distributed training on various hardware such as GPUs, Apple Silicon devices etc during training. -Use 🤗 Accelerate for inferencing on consumer hardware with small resources. +You can use 🤗 Accelerate for distributed training on various hardware such as GPUs, or Apple Silicon devices during training, and for inference on consumer hardware with fewer resources. -### Example of PEFT model training using 🤗 Accelerate's DeepSpeed integration +### Train with 🤗 Accelerate's DeepSpeed integration -DeepSpeed version required `v0.8.0`. An example is provided in `~examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py`. - a. First, run `accelerate config --config_file ds_zero3_cpu.yaml` and answer the questionnaire. - Below are the contents of the config file. +You'll need DeepSpeed version `v0.8.0` for this example. Feel free to check out the full example [script](https://github.com/huggingface/peft/blob/main/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py) for more details! + +1. Run `accelerate config --config_file ds_zero3_cpu.yaml` and answer the questionnaire to setup your environment. +Below are the contents of the config file. ```yaml compute_environment: LOCAL_MACHINE deepspeed_config: @@ -133,12 +132,12 @@ DeepSpeed version required `v0.8.0`. An example is provided in `~examples/condit same_network: true use_cpu: false ``` - b. run the below command to launch the example script +2. Run the following command to launch the example script: ```bash accelerate launch --config_file ds_zero3_cpu.yaml examples/peft_lora_seq2seq_accelerate_ds_zero3_offload.py ``` - c. output logs: +You'll see some output logs that look like this: ```bash GPU Memory before entering the train : 1916 GPU Memory consumed at the end of the train (end-begin): 66 @@ -163,7 +162,7 @@ DeepSpeed version required `v0.8.0`. An example is provided in `~examples/condit dataset['train'][label_column][:10]=['no complaint', 'no complaint', 'complaint', 'complaint', 'no complaint', 'no complaint', 'no complaint', 'complaint', 'complaint', 'no complaint'] ``` -### Example of PEFT model inference using 🤗 Accelerate's Big Model Inferencing capabilities +### Inference with 🤗 Accelerate's Big Model Inference An example is provided in `~examples/causal_language_modeling/peft_lora_clm_accelerate_big_model_inference.ipynb`. ## Model Support matrix From 891584c8d93dc0c9aeff578be5f656fb25d43745 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 28 Mar 2023 13:55:43 +0000 Subject: [PATCH 03/30] fix ci dreambooth --- examples/lora_dreambooth/train_dreambooth.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/lora_dreambooth/train_dreambooth.py b/examples/lora_dreambooth/train_dreambooth.py index 9145eca..32f78a8 100644 --- a/examples/lora_dreambooth/train_dreambooth.py +++ b/examples/lora_dreambooth/train_dreambooth.py @@ -1063,7 +1063,9 @@ def main(args): ) text_encoder_state_dict = {f"text_encoder_{k}": v for k, v in text_encoder_state_dict.items()} state_dict.update(text_encoder_state_dict) - lora_config["text_encoder_peft_config"] = unwarpped_text_encoder.get_peft_config_as_dict(inference=True) + lora_config["text_encoder_peft_config"] = unwarpped_text_encoder.get_peft_config_as_dict( + inference=True + ) accelerator.print(state_dict) accelerator.save(state_dict, os.path.join(args.output_dir, f"{args.instance_prompt}_lora.pt")) From 4626b36e273470884041f4ca84f44930c16c59f3 Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Wed, 29 Mar 2023 17:12:32 +0530 Subject: [PATCH 04/30] addressing remaining comments --- docs/{ => source}/_toctree.yml | 0 docs/{ => source}/index.mdx | 18 ------------------ docs/{ => source}/install.mdx | 0 docs/{ => source}/package_reference/config | 0 docs/{ => source}/package_reference/peft_model | 0 docs/{ => source}/package_reference/tuners | 0 docs/{ => source}/quicktour.mdx | 3 --- 7 files changed, 21 deletions(-) rename docs/{ => source}/_toctree.yml (100%) rename docs/{ => source}/index.mdx (75%) rename docs/{ => source}/install.mdx (100%) rename docs/{ => source}/package_reference/config (100%) rename docs/{ => source}/package_reference/peft_model (100%) rename docs/{ => source}/package_reference/tuners (100%) rename docs/{ => source}/quicktour.mdx (98%) diff --git a/docs/_toctree.yml b/docs/source/_toctree.yml similarity index 100% rename from docs/_toctree.yml rename to docs/source/_toctree.yml diff --git a/docs/index.mdx b/docs/source/index.mdx similarity index 75% rename from docs/index.mdx rename to docs/source/index.mdx index 4f5776f..008be12 100644 --- a/docs/index.mdx +++ b/docs/source/index.mdx @@ -24,21 +24,3 @@ Supported methods include: 2. Prefix Tuning: [Prefix-Tuning: Optimizing Continuous Prompts for Generation](https://aclanthology.org/2021.acl-long.353/), [P-Tuning v2: Prompt Tuning Can Be Comparable to Fine-tuning Universally Across Scales and Tasks](https://arxiv.org/pdf/2110.07602.pdf) 3. P-Tuning: [GPT Understands, Too](https://arxiv.org/pdf/2103.10385.pdf) 4. Prompt Tuning: [The Power of Scale for Parameter-Efficient Prompt Tuning](https://arxiv.org/pdf/2104.08691.pdf) - -## Getting started - -```python -from transformers import AutoModelForSeq2SeqLM -from peft import get_peft_config, get_peft_model, LoraConfig, TaskType - -model_name_or_path = "bigscience/mt0-large" -tokenizer_name_or_path = "bigscience/mt0-large" - -peft_config = LoraConfig(task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1) - -model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path) -model = get_peft_model(model, peft_config) -model.print_trainable_parameters() -# output: trainable params: 2359296 || all params: 1231940608 || trainable%: 0.19151053100118282 -``` - diff --git a/docs/install.mdx b/docs/source/install.mdx similarity index 100% rename from docs/install.mdx rename to docs/source/install.mdx diff --git a/docs/package_reference/config b/docs/source/package_reference/config similarity index 100% rename from docs/package_reference/config rename to docs/source/package_reference/config diff --git a/docs/package_reference/peft_model b/docs/source/package_reference/peft_model similarity index 100% rename from docs/package_reference/peft_model rename to docs/source/package_reference/peft_model diff --git a/docs/package_reference/tuners b/docs/source/package_reference/tuners similarity index 100% rename from docs/package_reference/tuners rename to docs/source/package_reference/tuners diff --git a/docs/quicktour.mdx b/docs/source/quicktour.mdx similarity index 98% rename from docs/quicktour.mdx rename to docs/source/quicktour.mdx index e0eb37f..a625aa8 100644 --- a/docs/quicktour.mdx +++ b/docs/source/quicktour.mdx @@ -29,9 +29,6 @@ from peft import LoraConfig, TaskType peft_config = LoraConfig(task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1) ``` -Here, `task_type` is the type of task you are training your model for. -For available task types, please refer [TaskType](package_reference/config#peft.config.TaskType). - 2. Load the base model you want to fine-tune. ```python From d8d1007732c464e4d86171f4741f8f2d1920d276 Mon Sep 17 00:00:00 2001 From: Vineet Kumar Date: Wed, 29 Mar 2023 18:50:14 +0530 Subject: [PATCH 05/30] Causal LM generation fix for prefix tuning: GPT2 model (#222) * expand attention mask after preparing generation inputs for prefix tuning * reformat * Update src/peft/peft_model.py Co-authored-by: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> * reformat as per black --------- Co-authored-by: Vineet Kumar Co-authored-by: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> --- src/peft/peft_model.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py index f73a66a..7491342 100644 --- a/src/peft/peft_model.py +++ b/src/peft/peft_model.py @@ -582,7 +582,13 @@ class PeftModelForCausalLM(PeftModel): else: if "input_ids" not in kwargs: raise ValueError("input_ids must be provided for Peft model generation") - if kwargs.get("attention_mask", None) is not None: + # For gpt2 models, we construct postion_ids on the fly by using attention mask, and position ids need to match input_shape. + # for prefix tuning, input shape is determined using `input_ids`. Thus we should not expand 'attention_mask' here + # for prompt tuning input_ids is not passed but a concatenated input_embeds is passed. Thus attention_mask needs to be of same size of num_virtual_tokens + input_ids + if kwargs.get("attention_mask", None) is not None and self.peft_config.peft_type in [ + PeftType.PROMPT_TUNING, + PeftType.P_TUNING, + ]: # concat prompt attention mask prefix_attention_mask = torch.ones( kwargs["input_ids"].shape[0], self.peft_config.num_virtual_tokens @@ -611,6 +617,14 @@ class PeftModelForCausalLM(PeftModel): def prepare_inputs_for_generation(self, *args, **kwargs): model_kwargs = self.base_model_prepare_inputs_for_generation(*args, **kwargs) if isinstance(self.peft_config, PromptLearningConfig): + if self.peft_config.peft_type == PeftType.PREFIX_TUNING: + prefix_attention_mask = torch.ones( + model_kwargs["input_ids"].shape[0], self.peft_config.num_virtual_tokens + ).to(model_kwargs["input_ids"].device) + model_kwargs["attention_mask"] = torch.cat( + (prefix_attention_mask, model_kwargs["attention_mask"]), dim=1 + ) + if model_kwargs["past_key_values"] is None and self.peft_config.peft_type == PeftType.PREFIX_TUNING: past_key_values = self.get_prompt(batch_size=model_kwargs["input_ids"].shape[0]) model_kwargs["past_key_values"] = past_key_values From df71b84341ae1ab3bc9b0d5f906d7a524850b63b Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Wed, 29 Mar 2023 15:28:38 +0200 Subject: [PATCH 06/30] [`CI`] Add more ci tests (#223) * add more tests * fix * add generate tests * make style * fix test * add -n * skip llama --- Makefile | 2 +- tests/test_peft_model.py | 39 ++++++++++++++++++++++++++++++++++----- tests/testing_common.py | 11 ++++++----- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 61549db..03ae1e0 100644 --- a/Makefile +++ b/Makefile @@ -17,4 +17,4 @@ style: doc-builder style src tests --max_len 119 test: - pytest tests/ \ No newline at end of file + pytest -n 3 tests/ \ No newline at end of file diff --git a/tests/test_peft_model.py b/tests/test_peft_model.py index 2ca4895..275a3cf 100644 --- a/tests/test_peft_model.py +++ b/tests/test_peft_model.py @@ -31,8 +31,14 @@ from .testing_common import PeftTestConfigManager # This has to be in the order: model_id, lora_kwargs, prefix_tuning_kwargs, prompt_encoder_kwargs, prompt_tuning_kwargs -PEFT_MODELS_TO_TEST = [ - ("hf-internal-testing/tiny-random-OPTForCausalLM", {"target_modules": ["q_proj", "v_proj"]}, {}, {}, {}), +PEFT_DECODER_MODELS_TO_TEST = [ + # ("HuggingFaceM4/tiny-random-LlamaForCausalLM", {}, {}, {}, {}), wait until the next `transformers` release + ("hf-internal-testing/tiny-random-OPTForCausalLM", {}, {}, {}, {}), + ("hf-internal-testing/tiny-random-GPTNeoXForCausalLM", {}, {}, {}, {}), + ("hf-internal-testing/tiny-random-GPT2LMHeadModel", {}, {}, {}, {}), + ("hf-internal-testing/tiny-random-BloomForCausalLM", {}, {}, {}, {}), + ("hf-internal-testing/tiny-random-gpt_neo", {}, {}, {}, {}), + ("hf-internal-testing/tiny-random-GPTJForCausalLM", {}, {}, {}, {}), ] @@ -48,7 +54,7 @@ class PeftModelTester(unittest.TestCase, PeftTestMixin): We use parametrized.expand for debugging purposes to test each model individually. """ - @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_MODELS_TO_TEST)) + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_DECODER_MODELS_TO_TEST)) def test_attributes_parametrized(self, test_name, model_id, config_cls, config_kwargs): self._test_model_attr(model_id, config_cls, config_kwargs) @@ -105,7 +111,7 @@ class PeftModelTester(unittest.TestCase, PeftTestMixin): self.assertTrue(dummy_output.requires_grad) - @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_MODELS_TO_TEST)) + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_DECODER_MODELS_TO_TEST)) def test_prepare_for_training_parametrized(self, test_name, model_id, config_cls, config_kwargs): self._test_prepare_for_training(model_id, config_cls, config_kwargs) @@ -151,6 +157,29 @@ class PeftModelTester(unittest.TestCase, PeftTestMixin): # check if `config.json` is not present self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json"))) - @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_MODELS_TO_TEST)) + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_DECODER_MODELS_TO_TEST)) def test_save_pretrained(self, test_name, model_id, config_cls, config_kwargs): self._test_save_pretrained(model_id, config_cls, config_kwargs) + + def _test_generate(self, model_id, config_cls, config_kwargs): + model = AutoModelForCausalLM.from_pretrained(model_id) + config = config_cls( + base_model_name_or_path=model_id, + **config_kwargs, + ) + model = get_peft_model(model, config) + model = model.to(self.torch_device) + + input_ids = torch.LongTensor([[1, 1, 1], [2, 1, 2]]).to(self.torch_device) + attention_mask = torch.LongTensor([[1, 1, 1], [1, 0, 1]]).to(self.torch_device) + + # check if `generate` works + _ = model.generate(input_ids=input_ids, attention_mask=attention_mask) + + with self.assertRaises(TypeError): + # check if `generate` raises an error if no positional arguments are passed + _ = model.generate(input_ids, attention_mask=attention_mask) + + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_DECODER_MODELS_TO_TEST)) + def test_generate(self, test_name, model_id, config_cls, config_kwargs): + self._test_generate(model_id, config_cls, config_kwargs) diff --git a/tests/testing_common.py b/tests/testing_common.py index dfdf1d8..96c0fdb 100644 --- a/tests/testing_common.py +++ b/tests/testing_common.py @@ -79,23 +79,24 @@ class ClassInstantier(OrderedDict): for model_tuple in model_list: model_id, lora_kwargs, prefix_tuning_kwargs, prompt_encoder_kwargs, prompt_tuning_kwargs = model_tuple for key, value in self.items(): + peft_method = value[1].copy() if key == "lora": # update value[1] if necessary if lora_kwargs is not None: - value[1].update(lora_kwargs) + peft_method.update(lora_kwargs) elif key == "prefix_tuning": # update value[1] if necessary if prefix_tuning_kwargs is not None: - value[1].update(prefix_tuning_kwargs) + peft_method.update(prefix_tuning_kwargs) elif key == "prompt_encoder": # update value[1] if necessary if prompt_encoder_kwargs is not None: - value[1].update(prompt_encoder_kwargs) + peft_method.update(prompt_encoder_kwargs) else: # update value[1] if necessary if prompt_tuning_kwargs is not None: - value[1].update(prompt_tuning_kwargs) - grid_parameters.append((f"test_{model_id}_{key}", model_id, value[0], value[1])) + peft_method.update(prompt_tuning_kwargs) + grid_parameters.append((f"test_{model_id}_{key}", model_id, value[0], peft_method)) return grid_parameters From d6c68ae1a5c17e3b4f1805233db3479d6033eaeb Mon Sep 17 00:00:00 2001 From: Aitor Gamarra <60578201+aitor-gamarra@users.noreply.github.com> Date: Wed, 29 Mar 2023 21:03:39 +0200 Subject: [PATCH 07/30] Show CONFIG_NAME instead of "config.json" --- src/peft/utils/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/peft/utils/config.py b/src/peft/utils/config.py index 3e2cf5b..2be3817 100644 --- a/src/peft/utils/config.py +++ b/src/peft/utils/config.py @@ -98,7 +98,7 @@ class PeftConfigMixin(PushToHubMixin): try: config_file = hf_hub_download(pretrained_model_name_or_path, CONFIG_NAME) except Exception: - raise ValueError(f"Can't find config.json at '{pretrained_model_name_or_path}'") + raise ValueError(f"Can't find '{CONFIG_NAME}' at '{pretrained_model_name_or_path}'") loaded_attributes = cls.from_json_file(config_file) From 8f63f565c6baa93de4bd57c21d38e0ce4868c519 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Thu, 30 Mar 2023 13:45:37 +0200 Subject: [PATCH 08/30] [`utils`] add merge_lora utility function (#227) * add merge_lora utility function * forward contrib credits from original script * some changes * make style * fix tets * finally fix tests * Update tests/test_peft_model.py * adapt from suggestions * adapt * Update src/peft/tuners/lora.py Co-authored-by: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> * fix 8bit * Update src/peft/tuners/lora.py Co-authored-by: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> --------- Co-authored-by: edbeeching Co-authored-by: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> --- src/peft/tuners/lora.py | 47 ++++++++++++++++++++++- tests/test_peft_model.py | 83 ++++++++++++++++++++++++++++++++-------- tests/testing_common.py | 63 ++++++++++++++++++------------ 3 files changed, 151 insertions(+), 42 deletions(-) diff --git a/src/peft/tuners/lora.py b/src/peft/tuners/lora.py index 0f65cbf..47f2c02 100644 --- a/src/peft/tuners/lora.py +++ b/src/peft/tuners/lora.py @@ -82,6 +82,10 @@ class LoraConfig(PeftConfig): "the final layer `classifier/score` are randomly initialized and as such need to be trainable and saved." }, ) + init_lora_weights: bool = field( + default=True, + metadata={"help": "Whether to initialize the weights of the Lora layers."}, + ) def __post_init__(self): self.peft_type = PeftType.LORA @@ -135,6 +139,7 @@ class LoraModel(torch.nn.Module): "fan_in_fan_out": self.peft_config.fan_in_fan_out, "merge_weights": (self.peft_config.merge_weights or self.peft_config.inference_mode) and not is_hf_device_map_available, + "init_lora_weights": self.peft_config.init_lora_weights, } key_list = [key for key, _ in self.model.named_modules()] for key in key_list: @@ -233,6 +238,37 @@ class LoraModel(torch.nn.Module): def disable_adapter_layers(self): self._set_adapter_layers(enabled=False) + def merge_and_unload(self): + r""" + This method merges the LoRa layers into the base model. This is needed if someone wants to use the base model + as a standalone model. + """ + if self.config.model_type == "gpt2": + raise ValueError("GPT2 models are not supported for merging LORA layers") + + if getattr(self.model, "is_loaded_in_8bit", False): + raise ValueError("Cannot merge LORA layers when the model is loaded in 8-bit mode") + + key_list = [key for key, _ in self.model.named_modules() if "lora" not in key] + for key in key_list: + parent, target, target_name = self._get_submodules(key) + if isinstance(target, LoraLayer): + bias = target.bias is not None + new_module = torch.nn.Linear(target.in_features, target.out_features, bias=bias) + + # manually merge if not merged + if not target.merged: + # merge weights per: https://arxiv.org/pdf/2106.09685.pdf / page 4 + if target.r > 0: + target.weight.data += ( + transpose(target.lora_B.weight @ target.lora_A.weight, target.fan_in_fan_out) + * target.scaling + ).to(target.weight.dtype) + target.merged = True + + self._replace_module(parent, target_name, new_module, target) + return self.model + # Below code is based on https://github.com/microsoft/LoRA/blob/main/loralib/layers.py # and modified to work with PyTorch FSDP @@ -297,6 +333,8 @@ class Linear(nn.Linear, LoraLayer): merge_weights: bool = True, **kwargs, ): + init_lora_weights = kwargs.pop("init_lora_weights", True) + nn.Linear.__init__(self, in_features, out_features, **kwargs) LoraLayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, merge_weights=merge_weights) @@ -308,7 +346,8 @@ class Linear(nn.Linear, LoraLayer): self.scaling = self.lora_alpha / self.r # Freezing the pre-trained weight matrix self.weight.requires_grad = False - self.reset_parameters() + if init_lora_weights: + self.reset_parameters() if fan_in_fan_out: self.weight.data = self.weight.data.T @@ -375,6 +414,8 @@ class MergedLinear(nn.Linear, LoraLayer): merge_weights: bool = True, **kwargs, ): + init_lora_weights = kwargs.pop("init_lora_weights", True) + nn.Linear.__init__(self, in_features, out_features, **kwargs) LoraLayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, merge_weights=merge_weights) if out_features % len(enable_lora) != 0: @@ -398,7 +439,9 @@ class MergedLinear(nn.Linear, LoraLayer): self.lora_ind = self.weight.new_zeros((out_features,), dtype=torch.bool).view(len(enable_lora), -1) self.lora_ind[enable_lora, :] = True self.lora_ind = self.lora_ind.view(-1) - self.reset_parameters() + + if init_lora_weights: + self.reset_parameters() if fan_in_fan_out: self.weight.data = self.weight.data.T diff --git a/tests/test_peft_model.py b/tests/test_peft_model.py index 275a3cf..4280ff3 100644 --- a/tests/test_peft_model.py +++ b/tests/test_peft_model.py @@ -30,17 +30,19 @@ from peft import ( from .testing_common import PeftTestConfigManager -# This has to be in the order: model_id, lora_kwargs, prefix_tuning_kwargs, prompt_encoder_kwargs, prompt_tuning_kwargs PEFT_DECODER_MODELS_TO_TEST = [ - # ("HuggingFaceM4/tiny-random-LlamaForCausalLM", {}, {}, {}, {}), wait until the next `transformers` release - ("hf-internal-testing/tiny-random-OPTForCausalLM", {}, {}, {}, {}), - ("hf-internal-testing/tiny-random-GPTNeoXForCausalLM", {}, {}, {}, {}), - ("hf-internal-testing/tiny-random-GPT2LMHeadModel", {}, {}, {}, {}), - ("hf-internal-testing/tiny-random-BloomForCausalLM", {}, {}, {}, {}), - ("hf-internal-testing/tiny-random-gpt_neo", {}, {}, {}, {}), - ("hf-internal-testing/tiny-random-GPTJForCausalLM", {}, {}, {}, {}), + "hf-internal-testing/tiny-random-OPTForCausalLM", + "hf-internal-testing/tiny-random-GPTNeoXForCausalLM", + "hf-internal-testing/tiny-random-GPT2LMHeadModel", + "hf-internal-testing/tiny-random-BloomForCausalLM", + "hf-internal-testing/tiny-random-gpt_neo", + "hf-internal-testing/tiny-random-GPTJForCausalLM", ] +FULL_GRID = { + "model_ids": PEFT_DECODER_MODELS_TO_TEST, +} + class PeftTestMixin: torch_device = "cuda" if torch.cuda.is_available() else "cpu" @@ -54,10 +56,6 @@ class PeftModelTester(unittest.TestCase, PeftTestMixin): We use parametrized.expand for debugging purposes to test each model individually. """ - @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_DECODER_MODELS_TO_TEST)) - def test_attributes_parametrized(self, test_name, model_id, config_cls, config_kwargs): - self._test_model_attr(model_id, config_cls, config_kwargs) - def _test_model_attr(self, model_id, config_cls, config_kwargs): model = AutoModelForCausalLM.from_pretrained(model_id) config = config_cls( @@ -70,6 +68,10 @@ class PeftModelTester(unittest.TestCase, PeftTestMixin): self.assertTrue(hasattr(model, "from_pretrained")) self.assertTrue(hasattr(model, "push_to_hub")) + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) + def test_attributes_parametrized(self, test_name, model_id, config_cls, config_kwargs): + self._test_model_attr(model_id, config_cls, config_kwargs) + def _test_prepare_for_training(self, model_id, config_cls, config_kwargs): model = AutoModelForCausalLM.from_pretrained(model_id).to(self.torch_device) config = config_cls( @@ -111,7 +113,7 @@ class PeftModelTester(unittest.TestCase, PeftTestMixin): self.assertTrue(dummy_output.requires_grad) - @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_DECODER_MODELS_TO_TEST)) + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) def test_prepare_for_training_parametrized(self, test_name, model_id, config_cls, config_kwargs): self._test_prepare_for_training(model_id, config_cls, config_kwargs) @@ -157,10 +159,61 @@ class PeftModelTester(unittest.TestCase, PeftTestMixin): # check if `config.json` is not present self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json"))) - @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_DECODER_MODELS_TO_TEST)) + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) def test_save_pretrained(self, test_name, model_id, config_cls, config_kwargs): self._test_save_pretrained(model_id, config_cls, config_kwargs) + def _test_merge_layers(self, model_id, config_cls, config_kwargs): + model = AutoModelForCausalLM.from_pretrained(model_id) + config = config_cls( + base_model_name_or_path=model_id, + **config_kwargs, + ) + model = get_peft_model(model, config) + model = model.to(self.torch_device) + + if config.peft_type != "LORA": + with self.assertRaises(AttributeError): + model = model.merge_and_unload() + elif model.config.model_type == "gpt2": + with self.assertRaises(ValueError): + model = model.merge_and_unload() + else: + dummy_input = torch.LongTensor([[1, 2, 3, 2, 1]]).to(self.torch_device) + model.eval() + logits_lora = model(dummy_input)[0] + + model = model.merge_and_unload() + + logits_merged = model(dummy_input)[0] + + transformers_model = AutoModelForCausalLM.from_pretrained(model_id).to(self.torch_device) + + logits_transformers = transformers_model(dummy_input)[0] + + self.assertTrue(torch.allclose(logits_lora, logits_merged, atol=1e-3, rtol=1e-3)) + self.assertFalse(torch.allclose(logits_merged, logits_transformers, atol=1e-3, rtol=1e-3)) + + with tempfile.TemporaryDirectory() as tmp_dirname: + model.save_pretrained(tmp_dirname) + + model_from_pretrained = AutoModelForCausalLM.from_pretrained(tmp_dirname).to(self.torch_device) + + logits_merged_from_pretrained = model_from_pretrained(dummy_input)[0] + + self.assertTrue(torch.allclose(logits_merged, logits_merged_from_pretrained, atol=1e-3, rtol=1e-3)) + + @parameterized.expand( + PeftTestConfigManager.get_grid_parameters( + { + "model_ids": PEFT_DECODER_MODELS_TO_TEST, + "lora_kwargs": {"init_lora_weights": [False], "merge_weights": [False, True]}, + }, + ) + ) + def test_merge_layers(self, test_name, model_id, config_cls, config_kwargs): + self._test_merge_layers(model_id, config_cls, config_kwargs) + def _test_generate(self, model_id, config_cls, config_kwargs): model = AutoModelForCausalLM.from_pretrained(model_id) config = config_cls( @@ -180,6 +233,6 @@ class PeftModelTester(unittest.TestCase, PeftTestMixin): # check if `generate` raises an error if no positional arguments are passed _ = model.generate(input_ids, attention_mask=attention_mask) - @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_DECODER_MODELS_TO_TEST)) + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(FULL_GRID)) def test_generate(self, test_name, model_id, config_cls, config_kwargs): self._test_generate(model_id, config_cls, config_kwargs) diff --git a/tests/testing_common.py b/tests/testing_common.py index 96c0fdb..633bb87 100644 --- a/tests/testing_common.py +++ b/tests/testing_common.py @@ -71,34 +71,47 @@ class ClassInstantier(OrderedDict): return super().__getitem__(key, *args, **kwargs) - def get_grid_parameters(self, model_list): + def get_grid_parameters(self, grid_parameters, filter_params_func=None): r""" Returns a list of all possible combinations of the parameters in the config classes. - """ - grid_parameters = [] - for model_tuple in model_list: - model_id, lora_kwargs, prefix_tuning_kwargs, prompt_encoder_kwargs, prompt_tuning_kwargs = model_tuple - for key, value in self.items(): - peft_method = value[1].copy() - if key == "lora": - # update value[1] if necessary - if lora_kwargs is not None: - peft_method.update(lora_kwargs) - elif key == "prefix_tuning": - # update value[1] if necessary - if prefix_tuning_kwargs is not None: - peft_method.update(prefix_tuning_kwargs) - elif key == "prompt_encoder": - # update value[1] if necessary - if prompt_encoder_kwargs is not None: - peft_method.update(prompt_encoder_kwargs) - else: - # update value[1] if necessary - if prompt_tuning_kwargs is not None: - peft_method.update(prompt_tuning_kwargs) - grid_parameters.append((f"test_{model_id}_{key}", model_id, value[0], peft_method)) - return grid_parameters + Args: + grid_parameters (`dict`): + A dictionary containing the parameters to be tested. There should be at least the key "model_ids" which + contains a list of model ids to be tested. The other keys should be the name of the config class + post-fixed with "_kwargs" and the value should be a dictionary containing the parameters to be tested + for that config class. + filter_params_func (`callable`, `optional`): + A function that takes a list of tuples and returns a list of tuples. This function is used to filter + out the tests that needs for example to be skipped. + + Returns: + generated_tests (`list`): + A list of tuples containing the name of the test, the model id, the config class and the config class + kwargs. + """ + generated_tests = [] + model_list = grid_parameters["model_ids"] + + for model_id in model_list: + for key, value in self.items(): + if "{}_kwargs".format(key) in grid_parameters: + peft_configs = [] + current_peft_config = value[1].copy() + for current_key, current_value in grid_parameters[f"{key}_kwargs"].items(): + for kwarg in current_value: + current_peft_config.update({current_key: kwarg}) + peft_configs.append(current_peft_config) + else: + peft_configs = [value[1].copy()] + + for peft_config in peft_configs: + generated_tests.append((f"test_{model_id}_{key}", model_id, value[0], peft_config)) + + if filter_params_func is not None: + generated_tests = filter_params_func(generated_tests) + + return generated_tests PeftTestConfigManager = ClassInstantier(CLASSES_MAPPING) From e4dcfaf1b356e399a7534a43afa2a78c1518b78a Mon Sep 17 00:00:00 2001 From: MKhalusova Date: Thu, 30 Mar 2023 11:30:55 -0400 Subject: [PATCH 09/30] task guide based on notebook --- docs/source/_toctree.yml | 4 + .../task_guides/image_classification_lora.mdx | 428 ++++++++++++++++++ 2 files changed, 432 insertions(+) create mode 100644 docs/source/task_guides/image_classification_lora.mdx diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index 211b83f..bb5a882 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -6,6 +6,10 @@ title: Quicktour - local: installation title: Installation +- title: Task Guides + sections: + - local: task_guides/image_classification_lora + title: Image classification using LoRA - title: Reference sections: - local: package_reference/peft_model diff --git a/docs/source/task_guides/image_classification_lora.mdx b/docs/source/task_guides/image_classification_lora.mdx new file mode 100644 index 0000000..17f00d3 --- /dev/null +++ b/docs/source/task_guides/image_classification_lora.mdx @@ -0,0 +1,428 @@ + + +# Fine-tuning for image classification using LoRA + +This guide demonstrates how to use LoRA, a low-rank approximation technique, to fine-tune an image classification model. +By using LoRA from 🤗 PEFT, we can reduce the number of trainable parameters in the model to only 0.77% of the original. + +LoRA achieves this reduction by adding low-rank "update matrices" to specific blocks of the model, such as the attention +blocks. During fine-tuning, only these matrices are trained, while the original model parameters are left unchanged. +At inference time, the update matrices are merged with the original model parameters to produce the final classification result. + +For more information on LoRA, please refer to the [original LoRA paper](https://arxiv.org/abs/2106.09685). + +## Install dependencies + +Install the libraries required for model training. To ensure you have access to all the latest features of 🤗 PEFT, +install it from source: + +```bash +pip install transformers accelerate evaluate datasets loralib git+https://github.com/huggingface/peft -q +``` + +Check the versions of all required libraries: + +```python +import transformers +import accelerate +import peft + +print(f"Transformers version: {transformers.__version__}") +print(f"Accelerate version: {accelerate.__version__}") +print(f"PEFT version: {peft.__version__}") +'Transformers version: 4.26.0' +'Accelerate version: 0.16.0' +'PEFT version: 0.1.0.dev0' +``` + +## Authenticate to share your model + +To share the fine-tuned model at the end of the training with the community, authenticate using your 🤗 token. +You can obtain your token from [here](https://huggingface.co/settings/token). + +```python +from huggingface_hub import notebook_login + +notebook_login() +``` + +## Select a model checkpoint to fine-tune + +Choose a model checkpoint from any of the model architectures supported for image classification. When in doubt, refer to +the [image classification task guide](https://huggingface.co/docs/transformers/v4.27.2/en/tasks/image_classification) in +🤗 Transformers documentation. + +```python +model_checkpoint = "google/vit-base-patch16-224-in21k" +``` + +## Load a dataset + +To keep this example's runtime short, let's only load the first 5000 instances from the training set of the Food-101 dataset: + +```python +from datasets import load_dataset + +dataset = load_dataset("food101", split="train[:5000]") +``` + +## Dataset Preparation + +To prepare the dataset for training and evaluation, create `label2id` and `id2label` dictionaries. These will come in +handy when performing inference and for metadata information: + +```python +labels = dataset.features["label"].names +label2id, id2label = dict(), dict() +for i, label in enumerate(labels): + label2id[label] = i + id2label[i] = label + +id2label[2] +'baklava' +``` + +Next, load the image processor of the model you're fine-tuning: + +```python +from transformers import AutoImageProcessor + +image_processor = AutoImageProcessor.from_pretrained(model_checkpoint) +``` + +The `image_processor` contains useful information on which size the training and evaluation images should be resized +to, as well as values that should be used to normalize the pixel values. Using the `image_processor`, prepare transformation +functions for the datasets. These functions will include data augmentation and pixel scaling: + +```python +from torchvision.transforms import ( + CenterCrop, + Compose, + Normalize, + RandomHorizontalFlip, + RandomResizedCrop, + Resize, + ToTensor, +) + +normalize = Normalize(mean=image_processor.image_mean, std=image_processor.image_std) +train_transforms = Compose( + [ + RandomResizedCrop(image_processor.size["height"]), + RandomHorizontalFlip(), + ToTensor(), + normalize, + ] +) + +val_transforms = Compose( + [ + Resize(image_processor.size["height"]), + CenterCrop(image_processor.size["height"]), + ToTensor(), + normalize, + ] +) + + +def preprocess_train(example_batch): + """Apply train_transforms across a batch.""" + example_batch["pixel_values"] = [train_transforms(image.convert("RGB")) for image in example_batch["image"]] + return example_batch + + +def preprocess_val(example_batch): + """Apply val_transforms across a batch.""" + example_batch["pixel_values"] = [val_transforms(image.convert("RGB")) for image in example_batch["image"]] + return example_batch +``` + +Split the dataset into training and validation sets: + +```python +splits = dataset.train_test_split(test_size=0.1) +train_ds = splits["train"] +val_ds = splits["test"] +``` + +Finally, set the transformation functions for the datasets accordingly: + +```python +train_ds.set_transform(preprocess_train) +val_ds.set_transform(preprocess_val) +``` + +## Load and prepare a model + +Before loading the model, let's define a helper function to check the total number of parameters a model has, as well +as how many of them are trainable. + +```python +def print_trainable_parameters(model): + trainable_params = 0 + all_param = 0 + for _, param in model.named_parameters(): + all_param += param.numel() + if param.requires_grad: + trainable_params += param.numel() + print( + f"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param:.2f}" + ) +``` + +It's important for to initialize the original model correctly as it will be used as a base to create a `PeftModel` you'll +actually fine-tune. Specify the `label2id` and `id2label` so that `AutoModelForImageClassification` can append a classification +head to the underlying model, adapted for this dataset. You should see the following output: + +``` +Some weights of ViTForImageClassification were not initialized from the model checkpoint at google/vit-base-patch16-224-in21k and are newly initialized: ['classifier.weight', 'classifier.bias'] +``` + +```python +from transformers import AutoModelForImageClassification, TrainingArguments, Trainer + +model = AutoModelForImageClassification.from_pretrained( + model_checkpoint, + label2id=label2id, + id2label=id2label, + ignore_mismatched_sizes=True, # provide this in case you're planning to fine-tune an already fine-tuned checkpoint +) +``` + +Before creating a `PeftModel`, you can check the number of trainable parameters in the original model: + +```python +print_trainable_parameters(model) +'trainable params: 85876325 || all params: 85876325 || trainable%: 100.00' +``` + +Next, use `PeftModel` to wrap the base model so that "update" matrices are added to the respective places. + +```python +from peft import LoraConfig, get_peft_model + +config = LoraConfig( + r=16, + lora_alpha=16, + target_modules=["query", "value"], + lora_dropout=0.1, + bias="none", + modules_to_save=["classifier"], +) +lora_model = get_peft_model(model, config) +print_trainable_parameters(lora_model) +'trainable params: 667493 || all params: 86466149 || trainable%: 0.77' +``` + +Let's unpack what's going on here. +To use LoRA, you need to specify the target modules to `LoraConfig` so that `get_peft_model()`` knows which modules +inside our model need to be amended with LoRA matrices. In this example, we're only interested in targeting the query and +value matrices of the attention blocks of the base model. Since the parameters corresponding to these matrices are "named" +with "query" and "value" respectively, we specify them accordingly in the `target_modules` argument of `LoraConfig`. + +We also specify `modules_to_save`. After wrapping the base model with `get_peft_model()` along with the `config`, we get +a new model where only the LoRA parameters are trainable (so-called "update matrices") while the pre-trained parameters +are kept frozen. However, we want the classifier parameters to be trained too when fine-tuning the base model on our +custom dataset. To ensure that the classifier parameters are also trained, we specify `modules_to_save`. This also +ensures that these modules are serialized alongside the LoRA trainable parameters when using utilities like `save_pretrained()` +and `push_to_hub()`. + +Here's what the other parameters mean: + +`r`: The dimension used by the LoRA update matrices. +`alpha`: Scaling factor. +`bias`: Specifies if the `bias` parameters should be trained. `None` denotes none of the `bias` parameters will be trained. + +`r` and `alpha` together control the total number of final trainable parameters when using LoRA, giving you the flexibility +to balance a trade-off between end performance and compute efficiency. + +By looking at the number of trainable parameters, you can see how many parameters we're actually training. Since the goal is +to achieve parameter-efficient fine-tuning, you should expect to see fewer trainable parameters in the `lora_model` +in comparison to the original model, which is indeed the case here. + +## Define training arguments + +For model fine-tuning, use [🤗 Trainer](https://huggingface.co/docs/transformers/main_classes/trainer). It accepts +several arguments which you can wrap using `TrainingArguments`. + +```python +from transformers import TrainingArguments, Trainer + + +model_name = model_checkpoint.split("/")[-1] +batch_size = 128 + +args = TrainingArguments( + f"{model_name}-finetuned-lora-food101", + remove_unused_columns=False, + evaluation_strategy="epoch", + save_strategy="epoch", + learning_rate=5e-3, + per_device_train_batch_size=batch_size, + gradient_accumulation_steps=4, + per_device_eval_batch_size=batch_size, + fp16=True, + num_train_epochs=5, + logging_steps=10, + load_best_model_at_end=True, + metric_for_best_model="accuracy", + push_to_hub=True, + label_names=["labels"], +) +``` + +Compared to fine-tuning the original model, you can use a larger batch size since there is only a handful of parameters to train. +You can also set a larger learning rate than the normal (1e-5 for example). + +This is a byproduct of the fact that the training affects only a small number of parameters. This can +potentially also reduce the need to conduct expensive hyperparameter tuning experiments. + +## Prepare evaluation metric + +```python +import numpy as np +import evaluate + +metric = evaluate.load("accuracy") + +# the compute_metrics function takes a Named Tuple as input: +# predictions, which are the logits of the model as Numpy arrays, +# and label_ids, which are the ground-truth labels as Numpy arrays. +def compute_metrics(eval_pred): + """Computes accuracy on a batch of predictions""" + predictions = np.argmax(eval_pred.predictions, axis=1) + return metric.compute(predictions=predictions, references=eval_pred.label_ids) + +``` + +## Define collation function + +A collation function is used by `Trainer` to gather a batch of training and evaluation examples and prepare them in a +format that is acceptable by the underlying model. + +```python +import torch + +def collate_fn(examples): + pixel_values = torch.stack([example["pixel_values"] for example in examples]) + labels = torch.tensor([example["label"] for example in examples]) + return {"pixel_values": pixel_values, "labels": labels} +``` + +## Train and evaluate + +Bring everything together - model, training arguments, data, collation function, etc. Then, start the training! + +```python +trainer = Trainer( + model, + args, + train_dataset=train_ds, + eval_dataset=val_ds, + tokenizer=image_processor, + compute_metrics=compute_metrics, + data_collator=collate_fn, +) +train_results = trainer.train() +``` + +In just a few minutes, the fine-tuned model shows 96% validation accuracy even on this small +subset of the training dataset. + +```python +trainer.evaluate(val_ds) +{'eval_loss': 0.14475855231285095, + 'eval_accuracy': 0.96, + 'eval_runtime': 3.5725, + 'eval_samples_per_second': 139.958, + 'eval_steps_per_second': 1.12, + 'epoch': 5.0} +``` + +## Share your model and run inference + +Once the fine-tuning is done, share the LoRA parameters with the community like so: + +```python +repo_name = f"sayakpaul/{model_name}-finetuned-lora-food101" +lora_model.push_to_hub(repo_name) +``` + +When calling `push_to_hub()` on the `lora_model`, only the LoRA parameters along with any modules specified in `modules_to_save` +are saved. Take a look at the [trained LoRA parameters](https://huggingface.co/sayakpaul/vit-base-patch16-224-in21k-finetuned-lora-food101/blob/main/adapter_model.bin). +You'll see that it's only 2.6 MB! This greatly helps with portability especially when using a very large model to fine-tune (such as [BLOOM](https://huggingface.co/bigscience/bloom). + +Next, let's see how to load the LoRA updated parameters along with our base model for inference. When you wrap a base model +with `PeftModel` that modifications are DONE in place. So to mitigate any concerns that might stem from in place modifications, +initialize the base model just like you did earlier and construct the inference model. + +```python +from peft import PeftConfig, PeftModel + + +config = PeftConfig.from_pretrained(repo_name) +model = model = AutoModelForImageClassification.from_pretrained( + config.base_model_name_or_path, + label2id=label2id, + id2label=id2label, + ignore_mismatched_sizes=True, # provide this in case you're planning to fine-tune an already fine-tuned checkpoint +) +# Load the LoRA model +inference_model = PeftModel.from_pretrained(model, repo_name) +``` + +Let's now fetch an example image for inference. + +```python +from PIL import Image +import requests + +url = "https://huggingface.co/datasets/sayakpaul/sample-datasets/resolve/main/beignets.jpeg" +image = Image.open(requests.get(url, stream=True).raw) +image +``` + +
+ image of beignets +
+ +First, instantiate an `image_processor` from the underlying model repo. + +```python +image_processor = AutoImageProcessor.from_pretrained(repo_name) +``` + +Then, prepare the example for inference. + +```python +encoding = image_processor(image.convert("RGB"), return_tensors="pt") +``` + +Finally, run inference! + +```python +with torch.no_grad(): + outputs = inference_model(**encoding) + logits = outputs.logits + +predicted_class_idx = logits.argmax(-1).item() +print("Predicted class:", inference_model.config.id2label[predicted_class_idx]) +'Predicted class: beignets' +``` + + + + + + + From d49cde41a7d62cc5e42b259430e95a6a1bc99e44 Mon Sep 17 00:00:00 2001 From: MKhalusova Date: Thu, 30 Mar 2023 12:09:28 -0400 Subject: [PATCH 10/30] make style --- .../task_guides/image_classification_lora.mdx | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/docs/source/task_guides/image_classification_lora.mdx b/docs/source/task_guides/image_classification_lora.mdx index 17f00d3..30cb3dc 100644 --- a/docs/source/task_guides/image_classification_lora.mdx +++ b/docs/source/task_guides/image_classification_lora.mdx @@ -40,9 +40,9 @@ import peft print(f"Transformers version: {transformers.__version__}") print(f"Accelerate version: {accelerate.__version__}") print(f"PEFT version: {peft.__version__}") -'Transformers version: 4.26.0' -'Accelerate version: 0.16.0' -'PEFT version: 0.1.0.dev0' +"Transformers version: 4.26.0" +"Accelerate version: 0.16.0" +"PEFT version: 0.1.0.dev0" ``` ## Authenticate to share your model @@ -89,7 +89,7 @@ for i, label in enumerate(labels): id2label[i] = label id2label[2] -'baklava' +"baklava" ``` Next, load the image processor of the model you're fine-tuning: @@ -203,7 +203,7 @@ Before creating a `PeftModel`, you can check the number of trainable parameters ```python print_trainable_parameters(model) -'trainable params: 85876325 || all params: 85876325 || trainable%: 100.00' +"trainable params: 85876325 || all params: 85876325 || trainable%: 100.00" ``` Next, use `PeftModel` to wrap the base model so that "update" matrices are added to the respective places. @@ -221,7 +221,7 @@ config = LoraConfig( ) lora_model = get_peft_model(model, config) print_trainable_parameters(lora_model) -'trainable params: 667493 || all params: 86466149 || trainable%: 0.77' +"trainable params: 667493 || all params: 86466149 || trainable%: 0.77" ``` Let's unpack what's going on here. @@ -295,6 +295,7 @@ import evaluate metric = evaluate.load("accuracy") + # the compute_metrics function takes a Named Tuple as input: # predictions, which are the logits of the model as Numpy arrays, # and label_ids, which are the ground-truth labels as Numpy arrays. @@ -302,7 +303,6 @@ def compute_metrics(eval_pred): """Computes accuracy on a batch of predictions""" predictions = np.argmax(eval_pred.predictions, axis=1) return metric.compute(predictions=predictions, references=eval_pred.label_ids) - ``` ## Define collation function @@ -313,6 +313,7 @@ format that is acceptable by the underlying model. ```python import torch + def collate_fn(examples): pixel_values = torch.stack([example["pixel_values"] for example in examples]) labels = torch.tensor([example["label"] for example in examples]) @@ -341,12 +342,14 @@ subset of the training dataset. ```python trainer.evaluate(val_ds) -{'eval_loss': 0.14475855231285095, - 'eval_accuracy': 0.96, - 'eval_runtime': 3.5725, - 'eval_samples_per_second': 139.958, - 'eval_steps_per_second': 1.12, - 'epoch': 5.0} +{ + "eval_loss": 0.14475855231285095, + "eval_accuracy": 0.96, + "eval_runtime": 3.5725, + "eval_samples_per_second": 139.958, + "eval_steps_per_second": 1.12, + "epoch": 5.0, +} ``` ## Share your model and run inference @@ -417,7 +420,7 @@ with torch.no_grad(): predicted_class_idx = logits.argmax(-1).item() print("Predicted class:", inference_model.config.id2label[predicted_class_idx]) -'Predicted class: beignets' +"Predicted class: beignets" ``` From 9ced552e65af707d8524a8e8f622c9f5475d09c4 Mon Sep 17 00:00:00 2001 From: MKhalusova Date: Thu, 30 Mar 2023 12:29:29 -0400 Subject: [PATCH 11/30] doc building fixes --- docs/source/_toctree.yml | 40 ++++++++++---------- docs/source/package_reference/config | 0 docs/source/package_reference/config.mdx | 1 + docs/source/package_reference/peft_model | 0 docs/source/package_reference/peft_model.mdx | 1 + docs/source/package_reference/tuners | 0 docs/source/package_reference/tuners.mdx | 1 + 7 files changed, 24 insertions(+), 19 deletions(-) delete mode 100644 docs/source/package_reference/config create mode 100644 docs/source/package_reference/config.mdx delete mode 100644 docs/source/package_reference/peft_model create mode 100644 docs/source/package_reference/peft_model.mdx delete mode 100644 docs/source/package_reference/tuners create mode 100644 docs/source/package_reference/tuners.mdx diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index bb5a882..1255844 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -1,20 +1,22 @@ -- title: Get Started - sections: - - local: index - title: 🤗 PEFT - - local: quicktour - title: Quicktour - - local: installation - title: Installation -- title: Task Guides - sections: - - local: task_guides/image_classification_lora - title: Image classification using LoRA +- title: Get started + sections: + - local: index + title: 🤗 PEFT + - local: quicktour + title: Quicktour + - local: install + title: Installation + +- title: Task guides + sections: + - local: task_guides/image_classification_lora + title: Image classification using LoRA + - title: Reference - sections: - - local: package_reference/peft_model - title: PEFT model - - local: package_reference/configs - title: Configuration - - local: package_reference/tuners - title: Tuners \ No newline at end of file + sections: + - local: package_reference/peft_model + title: PEFT model + - local: package_reference/config + title: Configuration + - local: package_reference/tuners + title: Tuners \ No newline at end of file diff --git a/docs/source/package_reference/config b/docs/source/package_reference/config deleted file mode 100644 index e69de29..0000000 diff --git a/docs/source/package_reference/config.mdx b/docs/source/package_reference/config.mdx new file mode 100644 index 0000000..af4abbf --- /dev/null +++ b/docs/source/package_reference/config.mdx @@ -0,0 +1 @@ +# Configuration \ No newline at end of file diff --git a/docs/source/package_reference/peft_model b/docs/source/package_reference/peft_model deleted file mode 100644 index e69de29..0000000 diff --git a/docs/source/package_reference/peft_model.mdx b/docs/source/package_reference/peft_model.mdx new file mode 100644 index 0000000..f6789d4 --- /dev/null +++ b/docs/source/package_reference/peft_model.mdx @@ -0,0 +1 @@ +# PEFT model \ No newline at end of file diff --git a/docs/source/package_reference/tuners b/docs/source/package_reference/tuners deleted file mode 100644 index e69de29..0000000 diff --git a/docs/source/package_reference/tuners.mdx b/docs/source/package_reference/tuners.mdx new file mode 100644 index 0000000..1e60f16 --- /dev/null +++ b/docs/source/package_reference/tuners.mdx @@ -0,0 +1 @@ +# Tuners \ No newline at end of file From 4d27c0c4672e07baf1d2ee738e13370fc8a64347 Mon Sep 17 00:00:00 2001 From: Guspan Tanadi <36249910+guspan-tanadi@users.noreply.github.com> Date: Fri, 31 Mar 2023 09:41:00 +0700 Subject: [PATCH 12/30] Have fix typo in README notebook provider name in capitalization --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3a9726e..5653c1a 100644 --- a/README.md +++ b/README.md @@ -126,14 +126,14 @@ Try out the 🤗 Gradio Space which should run seamlessly on a T4 instance: ![peft lora dreambooth gradio space](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/peft/peft_lora_dreambooth_gradio_space.png) ### Parameter Efficient Tuning of LLMs for RLHF components such as Ranker and Policy -- Here is an exmaple in [trl](https://github.com/lvwerra/trl) library using PEFT+INT8 for tuning policy model: [gpt2-sentiment_peft.py](https://github.com/lvwerra/trl/blob/main/examples/sentiment/scripts/gpt2-sentiment_peft.py) +- Here is an example in [trl](https://github.com/lvwerra/trl) library using PEFT+INT8 for tuning policy model: [gpt2-sentiment_peft.py](https://github.com/lvwerra/trl/blob/main/examples/sentiment/scripts/gpt2-sentiment_peft.py) - Example using PEFT for both reward model and policy [ToDo] ### INT8 training of large models in Colab using PEFT LoRA and bits_and_bytes -- Here is now a demo on how to fine tune [OPT-6.7b](https://huggingface.co/facebook/opt-6.7b) (14GB in fp16) in a Google colab: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1jCkpikz0J2o20FBQmYmAGdiKmJGOMo-o?usp=sharing) +- Here is now a demo on how to fine tune [OPT-6.7b](https://huggingface.co/facebook/opt-6.7b) (14GB in fp16) in a Google Colab: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1jCkpikz0J2o20FBQmYmAGdiKmJGOMo-o?usp=sharing) -- Here is now a demo on how to fine tune [whishper-large](openai/whisper-large-v2) (1.5B params) (14GB in fp16) in a Google colab: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1DOkD_5OUjFa0r5Ik3SgywJLJtEo2qLxO?usp=sharing) and [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1vhF8yueFqha3Y3CpTHN6q9EVcII9EYzs?usp=sharing) +- Here is now a demo on how to fine tune [whishper-large](openai/whisper-large-v2) (1.5B params) (14GB in fp16) in a Google Colab: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1DOkD_5OUjFa0r5Ik3SgywJLJtEo2qLxO?usp=sharing) and [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1vhF8yueFqha3Y3CpTHN6q9EVcII9EYzs?usp=sharing) ### Save compute and storage even for medium and small models @@ -143,7 +143,7 @@ performance comparable to full finetuning. An example of using LoRA for the task of adapting `LayoutLMForTokenClassification` on `FUNSD` dataset is given in `~examples/token_classification/PEFT_LoRA_LayoutLMForTokenClassification_on_FUNSD.py`. We can observe that with only `0.62 %` of parameters being trainable, we achieve performance (F1 0.777) comparable to full finetuning (F1 0.786) (without any hyerparam tuning runs for extracting more performance), and the checkpoint of this is only `2.8MB`. Now, if there are `N` such datasets, just have these PEFT models one for each dataset and save a lot of storage without having to worry about the problem of catastrophic forgetting or overfitting of backbone/base model. -Another example is fine-tuning [`roberta-large`](https://huggingface.co/roberta-large) on [`MRPC` GLUE](https://huggingface.co/datasets/glue/viewer/mrpc) dataset suing differenct PEFT methods. The notebooks are given in `~examples/sequence_classification`. +Another example is fine-tuning [`roberta-large`](https://huggingface.co/roberta-large) on [`MRPC` GLUE](https://huggingface.co/datasets/glue/viewer/mrpc) dataset using different PEFT methods. The notebooks are given in `~examples/sequence_classification`. ## PEFT + 🤗 Accelerate From 8a6004232ba0ef9658285aa5a035ababd489a9db Mon Sep 17 00:00:00 2001 From: Maria Khalusova Date: Fri, 31 Mar 2023 08:55:31 -0400 Subject: [PATCH 13/30] Apply suggestions from code review Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- .../task_guides/image_classification_lora.mdx | 49 +++++++++---------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/docs/source/task_guides/image_classification_lora.mdx b/docs/source/task_guides/image_classification_lora.mdx index 30cb3dc..4fca064 100644 --- a/docs/source/task_guides/image_classification_lora.mdx +++ b/docs/source/task_guides/image_classification_lora.mdx @@ -10,7 +10,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express o specific language governing permissions and limitations under the License. --> -# Fine-tuning for image classification using LoRA +# Image classification using LoRA This guide demonstrates how to use LoRA, a low-rank approximation technique, to fine-tune an image classification model. By using LoRA from 🤗 PEFT, we can reduce the number of trainable parameters in the model to only 0.77% of the original. @@ -27,10 +27,10 @@ Install the libraries required for model training. To ensure you have access to install it from source: ```bash -pip install transformers accelerate evaluate datasets loralib git+https://github.com/huggingface/peft -q +!pip install transformers accelerate evaluate datasets loralib git+https://github.com/huggingface/peft -q ``` -Check the versions of all required libraries: +Check the versions of all required libraries to make sure you are up to date: ```python import transformers @@ -48,7 +48,7 @@ print(f"PEFT version: {peft.__version__}") ## Authenticate to share your model To share the fine-tuned model at the end of the training with the community, authenticate using your 🤗 token. -You can obtain your token from [here](https://huggingface.co/settings/token). +You can obtain your token from your [account settings](https://huggingface.co/settings/token). ```python from huggingface_hub import notebook_login @@ -58,7 +58,7 @@ notebook_login() ## Select a model checkpoint to fine-tune -Choose a model checkpoint from any of the model architectures supported for image classification. When in doubt, refer to +Choose a model checkpoint from any of the model architectures supported for [image classification](https://huggingface.co/models?pipeline_tag=image-classification&sort=downloads). When in doubt, refer to the [image classification task guide](https://huggingface.co/docs/transformers/v4.27.2/en/tasks/image_classification) in 🤗 Transformers documentation. @@ -68,7 +68,7 @@ model_checkpoint = "google/vit-base-patch16-224-in21k" ## Load a dataset -To keep this example's runtime short, let's only load the first 5000 instances from the training set of the Food-101 dataset: +To keep this example's runtime short, let's only load the first 5000 instances from the training set of the [Food-101 dataset](https://huggingface.co/datasets/food101): ```python from datasets import load_dataset @@ -76,7 +76,7 @@ from datasets import load_dataset dataset = load_dataset("food101", split="train[:5000]") ``` -## Dataset Preparation +## Dataset preparation To prepare the dataset for training and evaluation, create `label2id` and `id2label` dictionaries. These will come in handy when performing inference and for metadata information: @@ -180,8 +180,8 @@ def print_trainable_parameters(model): ) ``` -It's important for to initialize the original model correctly as it will be used as a base to create a `PeftModel` you'll -actually fine-tune. Specify the `label2id` and `id2label` so that `AutoModelForImageClassification` can append a classification +It's important to initialize the original model correctly as it will be used as a base to create the `PeftModel` you'll +actually fine-tune. Specify the `label2id` and `id2label` so that [`~transformers.AutoModelForImageClassification`] can append a classification head to the underlying model, adapted for this dataset. You should see the following output: ``` @@ -206,7 +206,7 @@ print_trainable_parameters(model) "trainable params: 85876325 || all params: 85876325 || trainable%: 100.00" ``` -Next, use `PeftModel` to wrap the base model so that "update" matrices are added to the respective places. +Next, use `get_peft_model` to wrap the base model so that "update" matrices are added to the respective places. ```python from peft import LoraConfig, get_peft_model @@ -225,10 +225,10 @@ print_trainable_parameters(lora_model) ``` Let's unpack what's going on here. -To use LoRA, you need to specify the target modules to `LoraConfig` so that `get_peft_model()`` knows which modules +To use LoRA, you need to specify the target modules in `LoraConfig` so that `get_peft_model()` knows which modules inside our model need to be amended with LoRA matrices. In this example, we're only interested in targeting the query and value matrices of the attention blocks of the base model. Since the parameters corresponding to these matrices are "named" -with "query" and "value" respectively, we specify them accordingly in the `target_modules` argument of `LoraConfig`. +"query" and "value" respectively, we specify them accordingly in the `target_modules` argument of `LoraConfig`. We also specify `modules_to_save`. After wrapping the base model with `get_peft_model()` along with the `config`, we get a new model where only the LoRA parameters are trainable (so-called "update matrices") while the pre-trained parameters @@ -239,9 +239,9 @@ and `push_to_hub()`. Here's what the other parameters mean: -`r`: The dimension used by the LoRA update matrices. -`alpha`: Scaling factor. -`bias`: Specifies if the `bias` parameters should be trained. `None` denotes none of the `bias` parameters will be trained. +- `r`: The dimension used by the LoRA update matrices. +- `alpha`: Scaling factor. +- `bias`: Specifies if the `bias` parameters should be trained. `None` denotes none of the `bias` parameters will be trained. `r` and `alpha` together control the total number of final trainable parameters when using LoRA, giving you the flexibility to balance a trade-off between end performance and compute efficiency. @@ -252,8 +252,8 @@ in comparison to the original model, which is indeed the case here. ## Define training arguments -For model fine-tuning, use [🤗 Trainer](https://huggingface.co/docs/transformers/main_classes/trainer). It accepts -several arguments which you can wrap using `TrainingArguments`. +For model fine-tuning, use [`~transformers.Trainer`]. It accepts +several arguments which you can wrap using [`~transformers.TrainingArguments`]. ```python from transformers import TrainingArguments, Trainer @@ -281,11 +281,10 @@ args = TrainingArguments( ) ``` -Compared to fine-tuning the original model, you can use a larger batch size since there is only a handful of parameters to train. +Compared to non-PEFT methods, you can use a larger batch size since there are fewer parameters to train. You can also set a larger learning rate than the normal (1e-5 for example). -This is a byproduct of the fact that the training affects only a small number of parameters. This can -potentially also reduce the need to conduct expensive hyperparameter tuning experiments. +This can potentially also reduce the need to conduct expensive hyperparameter tuning experiments. ## Prepare evaluation metric @@ -307,7 +306,7 @@ def compute_metrics(eval_pred): ## Define collation function -A collation function is used by `Trainer` to gather a batch of training and evaluation examples and prepare them in a +A collation function is used by [`~transformers.Trainer`] to gather a batch of training and evaluation examples and prepare them in a format that is acceptable by the underlying model. ```python @@ -361,12 +360,12 @@ repo_name = f"sayakpaul/{model_name}-finetuned-lora-food101" lora_model.push_to_hub(repo_name) ``` -When calling `push_to_hub()` on the `lora_model`, only the LoRA parameters along with any modules specified in `modules_to_save` +When calling [`~transformers.PreTrainedModel.push_to_hub`] on the `lora_model`, only the LoRA parameters along with any modules specified in `modules_to_save` are saved. Take a look at the [trained LoRA parameters](https://huggingface.co/sayakpaul/vit-base-patch16-224-in21k-finetuned-lora-food101/blob/main/adapter_model.bin). -You'll see that it's only 2.6 MB! This greatly helps with portability especially when using a very large model to fine-tune (such as [BLOOM](https://huggingface.co/bigscience/bloom). +You'll see that it's only 2.6 MB! This greatly helps with portability, especially when using a very large model to fine-tune (such as [BLOOM](https://huggingface.co/bigscience/bloom)). Next, let's see how to load the LoRA updated parameters along with our base model for inference. When you wrap a base model -with `PeftModel` that modifications are DONE in place. So to mitigate any concerns that might stem from in place modifications, +with `PeftModel`, modifications are done *in-place*. To mitigate any concerns that might stem from in-place modifications, initialize the base model just like you did earlier and construct the inference model. ```python @@ -374,7 +373,7 @@ from peft import PeftConfig, PeftModel config = PeftConfig.from_pretrained(repo_name) -model = model = AutoModelForImageClassification.from_pretrained( +model = AutoModelForImageClassification.from_pretrained( config.base_model_name_or_path, label2id=label2id, id2label=id2label, From de2a46a2f9abbd653b350914dca1395c134f8943 Mon Sep 17 00:00:00 2001 From: MKhalusova Date: Fri, 31 Mar 2023 09:29:41 -0400 Subject: [PATCH 14/30] version fix --- docs/source/task_guides/image_classification_lora.mdx | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/source/task_guides/image_classification_lora.mdx b/docs/source/task_guides/image_classification_lora.mdx index 4fca064..8c82ea6 100644 --- a/docs/source/task_guides/image_classification_lora.mdx +++ b/docs/source/task_guides/image_classification_lora.mdx @@ -23,11 +23,10 @@ For more information on LoRA, please refer to the [original LoRA paper](https:// ## Install dependencies -Install the libraries required for model training. To ensure you have access to all the latest features of 🤗 PEFT, -install it from source: +Install the libraries required for model training: ```bash -!pip install transformers accelerate evaluate datasets loralib git+https://github.com/huggingface/peft -q +!pip install transformers accelerate evaluate datasets loralib peft -q ``` Check the versions of all required libraries to make sure you are up to date: @@ -40,9 +39,9 @@ import peft print(f"Transformers version: {transformers.__version__}") print(f"Accelerate version: {accelerate.__version__}") print(f"PEFT version: {peft.__version__}") -"Transformers version: 4.26.0" -"Accelerate version: 0.16.0" -"PEFT version: 0.1.0.dev0" +"Transformers version: 4.27.4" +"Accelerate version: 0.18.0" +"PEFT version: 0.2.0" ``` ## Authenticate to share your model From 221b39256db469607aa9557bab673414f17f1a55 Mon Sep 17 00:00:00 2001 From: MKhalusova Date: Fri, 31 Mar 2023 09:34:00 -0400 Subject: [PATCH 15/30] feedback addressed --- docs/source/task_guides/image_classification_lora.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/task_guides/image_classification_lora.mdx b/docs/source/task_guides/image_classification_lora.mdx index 8c82ea6..cfbbeca 100644 --- a/docs/source/task_guides/image_classification_lora.mdx +++ b/docs/source/task_guides/image_classification_lora.mdx @@ -294,15 +294,15 @@ import evaluate metric = evaluate.load("accuracy") -# the compute_metrics function takes a Named Tuple as input: -# predictions, which are the logits of the model as Numpy arrays, -# and label_ids, which are the ground-truth labels as Numpy arrays. def compute_metrics(eval_pred): """Computes accuracy on a batch of predictions""" predictions = np.argmax(eval_pred.predictions, axis=1) return metric.compute(predictions=predictions, references=eval_pred.label_ids) ``` +The `compute_metrics` function takes a named tuple as input: `predictions`, which are the logits of the model as Numpy arrays, +and `label_ids`, which are the ground-truth labels as Numpy arrays. + ## Define collation function A collation function is used by [`~transformers.Trainer`] to gather a batch of training and evaluation examples and prepare them in a From 39fb96316fa7c72cdfe95b9b7b63ff71fe85b942 Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 30 Mar 2023 14:16:01 -0700 Subject: [PATCH 16/30] first draft of api docs --- docs/source/_toctree.yml | 3 +- docs/source/package_reference/config.mdx | 19 +++++++++- docs/source/package_reference/peft_model.mdx | 37 +++++++++++++++++++- docs/source/package_reference/tuners.mdx | 36 ++++++++++++++++++- 4 files changed, 91 insertions(+), 4 deletions(-) diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index 1255844..12f8901 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -19,4 +19,5 @@ - local: package_reference/config title: Configuration - local: package_reference/tuners - title: Tuners \ No newline at end of file + title: Tuners + diff --git a/docs/source/package_reference/config.mdx b/docs/source/package_reference/config.mdx index af4abbf..1a2212f 100644 --- a/docs/source/package_reference/config.mdx +++ b/docs/source/package_reference/config.mdx @@ -1 +1,18 @@ -# Configuration \ No newline at end of file +# Configuration + +The configuration classes stores the configuration of a [`PeftModel`], PEFT adapter models, and the configurations of [`PrefixTuning`], [`PromptTuning`], and [`PromptEncoder`]. They contain methods for saving and loading model configurations from the Hub, specifying the PEFT method to use, type of task to perform, and model configurations like number of layers and number of attention heads. + +## PeftConfigMixin + +[[autodoc]] PeftConfigMixin + - all + +## PeftConfig + +[[autodoc]] PeftConfig + - all + +## PromptLearningConfig + +[[autodoc]] PromptLearningConfig + - all diff --git a/docs/source/package_reference/peft_model.mdx b/docs/source/package_reference/peft_model.mdx index f6789d4..771fd49 100644 --- a/docs/source/package_reference/peft_model.mdx +++ b/docs/source/package_reference/peft_model.mdx @@ -1 +1,36 @@ -# PEFT model \ No newline at end of file +## Models + +[`PeftModel`] is the base model class for specifying the base Transformer model and configuration to apply a PEFT method to. The base `PeftModel` contains methods for loading and saving models from the Hub, and supports the [`PromptEncoder`] for prompt learning. + +## PeftModel + +[[autodoc]] PeftModel + - all + +## PeftModelForSequenceClassification + +A `PeftModel` for sequence classification tasks. + +[[autodoc]] PeftModelForSequenceClassification + - all + +## PeftModelForTokenClassification + +A `PeftModel` for token classification tasks. + +[[autodoc]] PeftModelForTokenClassification + - all + +## PeftModelForCausalLM + +A `PeftModel` for causal language modeling. + +[[autodoc]] PeftModelForCausalLM + - all + +## PeftModelForSeq2SeqLM + +A `PeftModel` for sequence-to-sequence language modeling. + +[[autodoc]] PeftModelForSeq2SeqLM + - all diff --git a/docs/source/package_reference/tuners.mdx b/docs/source/package_reference/tuners.mdx index 1e60f16..93c0dc6 100644 --- a/docs/source/package_reference/tuners.mdx +++ b/docs/source/package_reference/tuners.mdx @@ -1 +1,35 @@ -# Tuners \ No newline at end of file +# Tuners + +Each tuner (or PEFT method) has a configuration and model. + +## LoRA + +For finetuning a model with LoRA. + +[[autodoc]] LoraConfig + +[[autodoc]] LoraModel + +[[autodoc]] LoraLayer + +[[autodoc]] Linear + +[[autodoc]] MergedLinear + +## P-tuning + +[[autodoc]] PromptEncoderConfig + +[[autodoc]] PromptEncoder + +## Prefix tuning + +[[autodoc]] PrefixTuningConfig + +[[autodoc]] PrefixEncoder + +## Prompt tuning + +[[autodoc]] PromptTuningConfig + +[[autodoc]] PromptEmbedding From 8fd53e004518cb52a7def340adfd0e873ae6298d Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 30 Mar 2023 14:26:12 -0700 Subject: [PATCH 17/30] fix path to peftconfigmixin? --- docs/source/package_reference/config.mdx | 2 +- docs/source/package_reference/peft_model.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/package_reference/config.mdx b/docs/source/package_reference/config.mdx index 1a2212f..6866685 100644 --- a/docs/source/package_reference/config.mdx +++ b/docs/source/package_reference/config.mdx @@ -4,7 +4,7 @@ The configuration classes stores the configuration of a [`PeftModel`], PEFT adap ## PeftConfigMixin -[[autodoc]] PeftConfigMixin +[[autodoc]] utils.config.PeftConfigMixin - all ## PeftConfig diff --git a/docs/source/package_reference/peft_model.mdx b/docs/source/package_reference/peft_model.mdx index 771fd49..f2618ef 100644 --- a/docs/source/package_reference/peft_model.mdx +++ b/docs/source/package_reference/peft_model.mdx @@ -1,4 +1,4 @@ -## Models +# Models [`PeftModel`] is the base model class for specifying the base Transformer model and configuration to apply a PEFT method to. The base `PeftModel` contains methods for loading and saving models from the Hub, and supports the [`PromptEncoder`] for prompt learning. From 47f05fe7b574130ba91beda623e0f9a8d261d243 Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 30 Mar 2023 14:31:57 -0700 Subject: [PATCH 18/30] fix path to loralayer too --- docs/source/package_reference/tuners.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/package_reference/tuners.mdx b/docs/source/package_reference/tuners.mdx index 93c0dc6..a44520c 100644 --- a/docs/source/package_reference/tuners.mdx +++ b/docs/source/package_reference/tuners.mdx @@ -10,7 +10,7 @@ For finetuning a model with LoRA. [[autodoc]] LoraModel -[[autodoc]] LoraLayer +[[autodoc]] tuners.lora.LoraLayer [[autodoc]] Linear From 7c31f5156723e56defae88fb760a193205a0b676 Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 30 Mar 2023 14:46:48 -0700 Subject: [PATCH 19/30] use explicit path --- docs/source/package_reference/tuners.mdx | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/source/package_reference/tuners.mdx b/docs/source/package_reference/tuners.mdx index a44520c..2ec0824 100644 --- a/docs/source/package_reference/tuners.mdx +++ b/docs/source/package_reference/tuners.mdx @@ -12,24 +12,24 @@ For finetuning a model with LoRA. [[autodoc]] tuners.lora.LoraLayer -[[autodoc]] Linear +[[autodoc]] tuners.lora.Linear -[[autodoc]] MergedLinear +[[autodoc]] tuners.lora.MergedLinear ## P-tuning -[[autodoc]] PromptEncoderConfig +[[autodoc]] tuners.p_tuning.PromptEncoderConfig -[[autodoc]] PromptEncoder +[[autodoc]] tuners.p_tuning.PromptEncoder ## Prefix tuning -[[autodoc]] PrefixTuningConfig +[[autodoc]] tuners.prefix_tuning.PrefixTuningConfig -[[autodoc]] PrefixEncoder +[[autodoc]] tuners.prefix_tuning.PrefixEncoder ## Prompt tuning -[[autodoc]] PromptTuningConfig +[[autodoc]] tuners.prompt_tuning.PromptTuningConfig -[[autodoc]] PromptEmbedding +[[autodoc]] tuners.prompt_tuning.PromptEmbedding \ No newline at end of file From 622a5a231ef36d2fa114032bcfe130c6299f9a24 Mon Sep 17 00:00:00 2001 From: Steven Liu Date: Fri, 31 Mar 2023 14:30:05 -0700 Subject: [PATCH 20/30] clean up docstrings --- src/peft/peft_model.py | 194 ++++++++++++++++++++----------- src/peft/tuners/lora.py | 39 ++++--- src/peft/tuners/p_tuning.py | 45 ++++--- src/peft/tuners/prefix_tuning.py | 36 +++--- src/peft/tuners/prompt_tuning.py | 44 ++++--- src/peft/utils/config.py | 12 +- 6 files changed, 235 insertions(+), 135 deletions(-) diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py index 7491342..2b79f4c 100644 --- a/src/peft/peft_model.py +++ b/src/peft/peft_model.py @@ -45,26 +45,26 @@ from .utils import ( class PeftModel(PushToHubMixin, torch.nn.Module): """ - Parameter-Efficient Fine-Tuning Model. Base model encompassing various Peft methods. + Base model encompassing various Peft methods. Args: - model ([`PreTrainedModel`]): The base transformer model used for Peft. + model ([`~transformers.PreTrainedModel`]): The base transformer model used for Peft. peft_config ([`PeftConfig`]): The configuration of the Peft model. **Attributes**: - - **base_model** ([`PreTrainedModel`]) -- The base transformer model used for Peft. + - **base_model** ([`~transformers.PreTrainedModel`]) -- The base transformer model used for Peft. - **peft_config** ([`PeftConfig`]) -- The configuration of the Peft model. - **modules_to_save** (`list` of `str`) -- The list of sub-module names to save when saving the model. - **prompt_encoder** ([`PromptEncoder`]) -- The prompt encoder used for Peft if - `isinstance(self.peft_config, PromptLearningConfig)`. + using [`PromptLearningConfig`]. - **prompt_tokens** (`torch.Tensor`) -- The virtual prompt tokens used for Peft if - `isinstance(self.peft_config, PromptLearningConfig)`. + using [`PromptLearningConfig`]. - **transformer_backbone_name** (`str`) -- The name of the transformer - backbone in the base model if `isinstance(self.peft_config, PromptLearningConfig)`. + backbone in the base model if using [`PromptLearningConfig`]. - **word_embeddings** (`torch.nn.Embedding`) -- The word embeddings of the transformer backbone - in the base model if `isinstance(self.peft_config, PromptLearningConfig)`. + in the base model if using [`PromptLearningConfig`]. """ def __init__(self, model, peft_config: PeftConfig): @@ -84,10 +84,11 @@ class PeftModel(PushToHubMixin, torch.nn.Module): def save_pretrained(self, save_directory, **kwargs): r""" - Args: This function saves the adapter model and the adapter configuration files to a directory, so that it can be - re-loaded using the `LoraModel.from_pretrained` class method, and also used by the `LoraModel.push_to_hub` + reloaded using the [`LoraModel.from_pretrained`] class method, and also used by the [`LoraModel.push_to_hub`] method. + + Args: save_directory (`str`): Directory where the adapter model and configuration files will be saved (will be created if it does not exist). @@ -117,17 +118,18 @@ class PeftModel(PushToHubMixin, torch.nn.Module): @classmethod def from_pretrained(cls, model, model_id, **kwargs): r""" + Instantiate a [`LoraModel`] from a pretrained Lora configuration and weights. + Args: - Instantiate a `LoraModel` from a pretrained Lora configuration and weights. - model (`transformers.PreTrainedModel`): - The model to be adapted. The model should be initialized with the `from_pretrained` method. from - `transformers` library. - model_id (`str`): + model ([`~transformers.PreTrainedModel`]): + The model to be adapted. The model should be initialized with the + [`~transformers.PreTrainedModel.from_pretrained`] method from the 🤗 Transformers library. + model_id (`str` or `os.PathLike`): The name of the Lora configuration to use. Can be either: - - A string, the `model id` of a Lora configuration hosted inside a model repo on - huggingface Hub - - A path to a directory containing a Lora configuration file saved using the - `save_pretrained` method, e.g., ``./my_lora_config_directory/``. + - A string, the `model id` of a Lora configuration hosted inside a model repo on the Hugging Face + Hub. + - A path to a directory containing a Lora configuration file saved using the `save_pretrained` + method (`./my_lora_config_directory/`). """ from .mapping import MODEL_TYPE_TO_PEFT_MODEL_MAPPING, PEFT_TYPE_TO_CONFIG_MAPPING @@ -322,25 +324,39 @@ class PeftModelForSequenceClassification(PeftModel): Peft model for sequence classification tasks. Args: - model ([`PreTrainedModel`]): Base transformer model + model ([`~transformers.PreTrainedModel`]): Base transformer model. peft_config ([`PeftConfig`]): Peft config. **Attributes**: - - **config** ([`PretrainedConfig`]) -- The configuration object of the base model. + - **config** ([`~transformers.PretrainedConfig`]) -- The configuration object of the base model. - **cls_layer_name** (`str`) -- The name of the classification layer. - Example:: + Example: - >>> from transformers import AutoModelForSequenceClassification >>> from peft import - PeftModelForSequenceClassification, get_peft_config >>> config = { - 'peft_type': 'PREFIX_TUNING', 'task_type': 'SEQ_CLS', 'inference_mode': False, 'num_virtual_tokens': - 20, 'token_dim': 768, 'num_transformer_submodules': 1, 'num_attention_heads': 12, 'num_layers': 12, - 'encoder_hidden_size': 768, 'prefix_projection': False, 'postprocess_past_key_value_function': None - } - >>> peft_config = get_peft_config(config) >>> model = - AutoModelForSequenceClassification.from_pretrained("bert-base-cased") >>> peft_model = - PeftModelForSequenceClassification(model, peft_config) >>> peft_model.print_trainable_parameters() trainable - params: 370178 || all params: 108680450 || trainable%: 0.3406113979101117 + ```py + >>> from transformers import AutoModelForSequenceClassification + >>> from peft import PeftModelForSequenceClassification, get_peft_config + + >>> config = { + ... "peft_type": "PREFIX_TUNING", + ... "task_type": "SEQ_CLS", + ... "inference_mode": False, + ... "num_virtual_tokens": 20, + ... "token_dim": 768, + ... "num_transformer_submodules": 1, + ... "num_attention_heads": 12, + ... "num_layers": 12, + ... "encoder_hidden_size": 768, + ... "prefix_projection": False, + ... "postprocess_past_key_value_function": None, + ... } + + >>> peft_config = get_peft_config(config) + >>> model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased") + >>> peft_model = PeftModelForSequenceClassification(model, peft_config) + >>> peft_model.print_trainable_parameters() + trainable params: 370178 || all params: 108680450 || trainable%: 0.3406113979101117 + ``` """ def __init__(self, model, peft_config: PeftConfig): @@ -490,24 +506,39 @@ class PeftModelForSequenceClassification(PeftModel): class PeftModelForCausalLM(PeftModel): """ - Peft model for Causal LM + Peft model for causal language modeling. Args: - model ([`PreTrainedModel`]): Base transformer model + model ([`~transformers.PreTrainedModel`]): Base transformer model. peft_config ([`PeftConfig`]): Peft config. - Example:: + Example: + + ```py + >>> from transformers import AutoModelForCausalLM + >>> from peft import PeftModelForCausalLM, get_peft_config - >>> from transformers import AutoModelForCausalLM >>> from peft import PeftModelForCausalLM, get_peft_config >>> config = { - 'peft_type': 'PREFIX_TUNING', 'task_type': 'CAUSAL_LM', 'inference_mode': False, 'num_virtual_tokens': - 20, 'token_dim': 1280, 'num_transformer_submodules': 1, 'num_attention_heads': 20, 'num_layers': 36, - 'encoder_hidden_size': 1280, 'prefix_projection': False, 'postprocess_past_key_value_function': None - } - >>> peft_config = get_peft_config(config) >>> model = AutoModelForCausalLM.from_pretrained("gpt2-large") >>> - peft_model = PeftModelForCausalLM(model, peft_config) >>> peft_model.print_trainable_parameters() trainable - params: 1843200 || all params: 775873280 || trainable%: 0.23756456724479544 + ... "peft_type": "PREFIX_TUNING", + ... "task_type": "CAUSAL_LM", + ... "inference_mode": False, + ... "num_virtual_tokens": 20, + ... "token_dim": 1280, + ... "num_transformer_submodules": 1, + ... "num_attention_heads": 20, + ... "num_layers": 36, + ... "encoder_hidden_size": 1280, + ... "prefix_projection": False, + ... "postprocess_past_key_value_function": None, + ... } + + >>> peft_config = get_peft_config(config) + >>> model = AutoModelForCausalLM.from_pretrained("gpt2-large") + >>> peft_model = PeftModelForCausalLM(model, peft_config) + >>> peft_model.print_trainable_parameters() + trainable params: 1843200 || all params: 775873280 || trainable%: 0.23756456724479544 + ``` """ def __init__(self, model, peft_config: PeftConfig): @@ -641,24 +672,39 @@ class PeftModelForCausalLM(PeftModel): class PeftModelForSeq2SeqLM(PeftModel): """ - Peft model for Seq2Seq LM + Peft model for sequence-to-sequence language modeling. Args: - model ([`PreTrainedModel`]): Base transformer model + model ([`~transformers.PreTrainedModel`]): Base transformer model. peft_config ([`PeftConfig`]): Peft config. - Example:: + Example: + + ```py + >>> from transformers import AutoModelForSeq2SeqLM + >>> from peft import PeftModelForSeq2SeqLM, get_peft_config - >>> from transformers import AutoModelForSeq2SeqLM >>> from peft import PeftModelForSeq2SeqLM, get_peft_config >>> config = { - 'peft_type': 'LORA', 'task_type': 'SEQ_2_SEQ_LM', 'inference_mode': False, 'r': 8, 'target_modules': - ['q', 'v'], 'lora_alpha': 32, 'lora_dropout': 0.1, 'merge_weights': False, 'fan_in_fan_out': False, - 'enable_lora': None, 'bias': 'none' - } - >>> peft_config = get_peft_config(config) >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") >>> - peft_model = PeftModelForSeq2SeqLM(model, peft_config) >>> peft_model.print_trainable_parameters() trainable - params: 884736 || all params: 223843584 || trainable%: 0.3952474242013566 + ... "peft_type": "LORA", + ... "task_type": "SEQ_2_SEQ_LM", + ... "inference_mode": False, + ... "r": 8, + ... "target_modules": ["q", "v"], + ... "lora_alpha": 32, + ... "lora_dropout": 0.1, + ... "merge_weights": False, + ... "fan_in_fan_out": False, + ... "enable_lora": None, + ... "bias": "none", + ... } + + >>> peft_config = get_peft_config(config) + >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") + >>> peft_model = PeftModelForSeq2SeqLM(model, peft_config) + >>> peft_model.print_trainable_parameters() + trainable params: 884736 || all params: 223843584 || trainable%: 0.3952474242013566 + ``` """ def __init__(self, model, peft_config: PeftConfig): @@ -808,28 +854,42 @@ class PeftModelForSeq2SeqLM(PeftModel): class PeftModelForTokenClassification(PeftModel): """ - Peft model for sequence classification tasks. + Peft model for token classification tasks. Args: - model ([`PreTrainedModel`]): Base transformer model + model ([`~transformers.PreTrainedModel`]): Base transformer model. peft_config ([`PeftConfig`]): Peft config. **Attributes**: - - **config** ([`PretrainedConfig`]) -- The configuration object of the base model. + - **config** ([`~transformers.PretrainedConfig`]) -- The configuration object of the base model. - **cls_layer_name** (`str`) -- The name of the classification layer. - Example:: + Example: - >>> from transformers import AutoModelForSequenceClassification >>> from peft import - PeftModelForTokenClassification, get_peft_config >>> config = { - 'peft_type': 'PREFIX_TUNING', 'task_type': 'TOKEN_CLS', 'inference_mode': False, 'num_virtual_tokens': - 20, 'token_dim': 768, 'num_transformer_submodules': 1, 'num_attention_heads': 12, 'num_layers': 12, - 'encoder_hidden_size': 768, 'prefix_projection': False, 'postprocess_past_key_value_function': None - } - >>> peft_config = get_peft_config(config) >>> model = - AutoModelForTokenClassification.from_pretrained("bert-base-cased") >>> peft_model = - PeftModelForTokenClassification(model, peft_config) >>> peft_model.print_trainable_parameters() trainable - params: 370178 || all params: 108680450 || trainable%: 0.3406113979101117 + ```py + >>> from transformers import AutoModelForSequenceClassification + >>> from peft import PeftModelForTokenClassification, get_peft_config + + >>> config = { + ... "peft_type": "PREFIX_TUNING", + ... "task_type": "TOKEN_CLS", + ... "inference_mode": False, + ... "num_virtual_tokens": 20, + ... "token_dim": 768, + ... "num_transformer_submodules": 1, + ... "num_attention_heads": 12, + ... "num_layers": 12, + ... "encoder_hidden_size": 768, + ... "prefix_projection": False, + ... "postprocess_past_key_value_function": None, + ... } + + >>> peft_config = get_peft_config(config) + >>> model = AutoModelForTokenClassification.from_pretrained("bert-base-cased") + >>> peft_model = PeftModelForTokenClassification(model, peft_config) + >>> peft_model.print_trainable_parameters() + trainable params: 370178 || all params: 108680450 || trainable%: 0.3406113979101117 + ``` """ def __init__(self, model, peft_config: PeftConfig): diff --git a/src/peft/tuners/lora.py b/src/peft/tuners/lora.py index 47f2c02..51cd56f 100644 --- a/src/peft/tuners/lora.py +++ b/src/peft/tuners/lora.py @@ -39,19 +39,19 @@ if is_bnb_available(): @dataclass class LoraConfig(PeftConfig): """ - This is the configuration class to store the configuration of a [`~peft.Lora`]. + This is the configuration class to store the configuration of a [`LoraModel`]. Args: - r (`int`): Lora attention dimension + r (`int`): Lora attention dimension. target_modules (`Union[List[str],str]`): The names of the modules to apply Lora to. lora_alpha (`float`): The alpha parameter for Lora scaling. lora_dropout (`float`): The dropout probability for Lora layers. merge_weights (`bool`): Whether to merge the weights of the Lora layers with the base transformer model in `eval` mode. - fan_in_fan_out (`bool`): Set this to True if the layer to replace stores weight like (fan_in, fan_out) - enable_lora ( `List[bool]`): Used with `lora.MergedLinear`. - bias (`str`): Bias type for Lora. Can be 'none', 'all' or 'lora_only' - modules_to_save (`List[str]`):List of modules apart from LoRA layers to be set as trainable + fan_in_fan_out (`bool`): Set this to True if the layer to replace stores weight like (`fan_in`, `fan_out`). + enable_lora ( `List[bool]`): Used with [`lora.MergedLinear`]. + bias (`str`): Bias type for Lora. Can be `none`, `all` or `lora_only`. + modules_to_save (`List[str]`): List of modules apart from Lora layers to be set as trainable and saved in the final checkpoint. """ @@ -96,22 +96,33 @@ class LoraModel(torch.nn.Module): Creates Low Rank Adapter (Lora) model from a pretrained transformers model. Args: - model ([`transformers.PreTrainedModel`]): The model to be adapted. + model ([`~transformers.PreTrainedModel`]): The model to be adapted. config ([`LoraConfig`]): The configuration of the Lora model. Returns: `torch.nn.Module`: The Lora model. - Example:: + Example: - >>> from transformers import AutoModelForSeq2SeqLM, LoraConfig >>> from peft import LoraModel, LoraConfig >>> - config = LoraConfig( - peft_type="LORA", task_type="SEQ_2_SEQ_LM", r=8, lora_alpha=32, target_modules=["q", "v"], - lora_dropout=0.01, ) - >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") >>> lora_model = LoraModel(config, model) + ```py + >>> from transformers import AutoModelForSeq2SeqLM, LoraConfig + >>> from peft import LoraModel, LoraConfig + + >>> config = LoraConfig( + ... peft_type="LORA", + ... task_type="SEQ_2_SEQ_LM", + ... r=8, + ... lora_alpha=32, + ... target_modules=["q", "v"], + ... lora_dropout=0.01, + ... ) + + >>> model = AutoModelForSeq2SeqLM.from_pretrained("t5-base") + >>> lora_model = LoraModel(config, model) + ``` **Attributes**: - - **model** ([`transformers.PreTrainedModel`]) -- The model to be adapted. + - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted. - **peft_config** ([`LoraConfig`]): The configuration of the Lora model. """ diff --git a/src/peft/tuners/p_tuning.py b/src/peft/tuners/p_tuning.py index b9c38c4..4a272f3 100644 --- a/src/peft/tuners/p_tuning.py +++ b/src/peft/tuners/p_tuning.py @@ -31,11 +31,11 @@ class PromptEncoderReparameterizationType(str, enum.Enum): @dataclass class PromptEncoderConfig(PromptLearningConfig): """ - This is the configuration class to store the configuration of a [`~peft.PromptEncoder`]. + This is the configuration class to store the configuration of a [`PromptEncoder`]. Args: - encoder_reparameterization_type - (Union[[`PromptEncoderReparameterizationType`], `str`]): The type of reparameterization to use. + encoder_reparameterization_type (Union[[`PromptEncoderReparameterizationType`], `str`]): + The type of reparameterization to use. encoder_hidden_size (`int`): The hidden size of the prompt encoder. encoder_num_layers (`int`): The number of layers of the prompt encoder. encoder_dropout (`float`): The dropout probability of the prompt encoder. @@ -71,19 +71,30 @@ class PromptEncoder(torch.nn.Module): Args: config ([`PromptEncoderConfig`]): The configuration of the prompt encoder. - Example:: + Example: - >>> from peft import PromptEncoder, PromptEncoderConfig >>> config = PromptEncoderConfig( - peft_type="P_TUNING", task_type="SEQ_2_SEQ_LM", num_virtual_tokens=20, token_dim=768, - num_transformer_submodules=1, num_attention_heads=12, num_layers=12, - encoder_reparameterization_type="MLP", encoder_hidden_size=768 - ) - >>> prompt_encoder = PromptEncoder(config) + ```py + >>> from peft import PromptEncoder, PromptEncoderConfig + + >>> config = PromptEncoderConfig( + ... peft_type="P_TUNING", + ... task_type="SEQ_2_SEQ_LM", + ... num_virtual_tokens=20, + ... token_dim=768, + ... num_transformer_submodules=1, + ... num_attention_heads=12, + ... num_layers=12, + ... encoder_reparameterization_type="MLP", + ... encoder_hidden_size=768, + ... ) + + >>> prompt_encoder = PromptEncoder(config) + ``` **Attributes**: - - **embedding** ([`~torch.nn.Embedding`]) -- The embedding layer of the prompt encoder. - - **mlp_head** ([`~torch.nn.Sequential`]) -- The MLP head of the prompt encoder if `inference_mode=False`. - - **lstm_head** ([`~torch.nn.LSTM`]) -- The LSTM head of the prompt encoder if `inference_mode=False` and + - **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prompt encoder. + - **mlp_head** (`torch.nn.Sequential`) -- The MLP head of the prompt encoder if `inference_mode=False`. + - **lstm_head** (`torch.nn.LSTM`) -- The LSTM head of the prompt encoder if `inference_mode=False` and `encoder_reparameterization_type="LSTM"`. - **token_dim** (`int`) -- The hidden embedding dimension of the base transformer model. - **input_size** (`int`) -- The input size of the prompt encoder. @@ -91,13 +102,13 @@ class PromptEncoder(torch.nn.Module): - **hidden_size** (`int`) -- The hidden size of the prompt encoder. - **total_virtual_tokens** (`int`): The total number of virtual tokens of the prompt encoder. - - **encoder_type** (Union[[`PromptEncoderReparameterizationType`], `str`]): - The encoder type of the prompt encoder. + - **encoder_type** (Union[[`PromptEncoderReparameterizationType`], `str`]): The encoder type of the prompt + encoder. - Input shape: (batch_size, total_virtual_tokens) + Input shape: (`batch_size`, `total_virtual_tokens`) - Output shape: (batch_size, total_virtual_tokens, token_dim) + Output shape: (`batch_size`, `total_virtual_tokens`, `token_dim`) """ def __init__(self, config): diff --git a/src/peft/tuners/prefix_tuning.py b/src/peft/tuners/prefix_tuning.py index fcb207c..d18000e 100644 --- a/src/peft/tuners/prefix_tuning.py +++ b/src/peft/tuners/prefix_tuning.py @@ -24,7 +24,7 @@ from ..utils import PeftType, PromptLearningConfig @dataclass class PrefixTuningConfig(PromptLearningConfig): """ - This is the configuration class to store the configuration of a [`~peft.PrefixEncoder`]. + This is the configuration class to store the configuration of a [`PrefixEncoder`]. Args: encoder_hidden_size (`int`): The hidden size of the prompt encoder. @@ -48,30 +48,38 @@ class PrefixTuningConfig(PromptLearningConfig): # with some refactor class PrefixEncoder(torch.nn.Module): r""" - The torch.nn model to encode the prefix + The `torch.nn` model to encode the prefix. Args: config ([`PrefixTuningConfig`]): The configuration of the prefix encoder. - Example:: + Example: - >>> from peft import PrefixEncoder, PrefixTuningConfig >>> config = PrefixTuningConfig( - peft_type="PREFIX_TUNING", task_type="SEQ_2_SEQ_LM", num_virtual_tokens=20, token_dim=768, - num_transformer_submodules=1, num_attention_heads=12, num_layers=12, encoder_hidden_size=768 - ) - >>> prefix_encoder = PrefixEncoder(config) + ```py + >>> from peft import PrefixEncoder, PrefixTuningConfig + >>> config = PrefixTuningConfig( + ... peft_type="PREFIX_TUNING", + ... task_type="SEQ_2_SEQ_LM", + ... num_virtual_tokens=20, + ... token_dim=768, + ... num_transformer_submodules=1, + ... num_attention_heads=12, + ... num_layers=12, + ... encoder_hidden_size=768, + ... ) + >>> prefix_encoder = PrefixEncoder(config) + ``` **Attributes**: - - **embedding** (`torch.nn.Embedding`) -- - The embedding layer of the prefix encoder. - - **transform** (`torch.nn.Sequential`) -- The - two-layer MLP to transform the prefix embeddings if `prefix_projection` is `True`. + - **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prefix encoder. + - **transform** (`torch.nn.Sequential`) -- The two-layer MLP to transform the prefix embeddings if + `prefix_projection` is `True`. - **prefix_projection** (`bool`) -- Whether to project the prefix embeddings. - Input shape: (batch_size, num_virtual_tokens) + Input shape: (`batch_size`, `num_virtual_tokens`) - Output shape: (batch_size, num_virtual_tokens, 2*layers*hidden) + Output shape: (`batch_size`, `num_virtual_tokens`, `2*layers*hidden`) """ def __init__(self, config): diff --git a/src/peft/tuners/prompt_tuning.py b/src/peft/tuners/prompt_tuning.py index 1dead1d..6880ff7 100644 --- a/src/peft/tuners/prompt_tuning.py +++ b/src/peft/tuners/prompt_tuning.py @@ -31,14 +31,14 @@ class PromptTuningInit(str, enum.Enum): @dataclass class PromptTuningConfig(PromptLearningConfig): """ - This is the configuration class to store the configuration of a [`~peft.PromptEmbedding`]. + This is the configuration class to store the configuration of a [`PromptEmbedding`]. Args: prompt_tuning_init (Union[[`PromptTuningInit`], `str`]): The initialization of the prompt embedding. - prompt_tuning_init_text ( Optional[`str`]): The text to initialize the prompt embedding. - Only used if `prompt_tuning_init` is `TEXT` - tokenizer_name_or_path ( Optional[`str`]): The name or path of the tokenizer. - Only used if `prompt_tuning_init` is `TEXT` + prompt_tuning_init_text (`str`, *optional*): + The text to initialize the prompt embedding. Only used if `prompt_tuning_init` is `TEXT`. + tokenizer_name_or_path (`str`, *optional*): + The name or path of the tokenizer. Only used if `prompt_tuning_init` is `TEXT`. """ prompt_tuning_init: Union[PromptTuningInit, str] = field( @@ -71,23 +71,33 @@ class PromptEmbedding(torch.nn.Module): word_embeddings (`torch.nn.Module`): The word embeddings of the base transformer model. **Attributes**: - **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prompt embedding. + - **embedding** (`torch.nn.Embedding`) -- The embedding layer of the prompt embedding. - Example:: + Example: - >>> from peft import PromptEmbedding, PromptTuningConfig >>> config = PromptTuningConfig( - peft_type="PROMPT_TUNING", task_type="SEQ_2_SEQ_LM", num_virtual_tokens=20, token_dim=768, - num_transformer_submodules=1, num_attention_heads=12, num_layers=12, prompt_tuning_init="TEXT", - prompt_tuning_init_text="Predict if sentiment of this review is positive, negative or neutral", - tokenizer_name_or_path="t5-base", - ) - >>> # t5_model.shared is the word embeddings of the base model >>> prompt_embedding = PromptEmbedding(config, - t5_model.shared) + ```py + >>> from peft import PromptEmbedding, PromptTuningConfig + >>> config = PromptTuningConfig( + ... peft_type="PROMPT_TUNING", + ... task_type="SEQ_2_SEQ_LM", + ... num_virtual_tokens=20, + ... token_dim=768, + ... num_transformer_submodules=1, + ... num_attention_heads=12, + ... num_layers=12, + ... prompt_tuning_init="TEXT", + ... prompt_tuning_init_text="Predict if sentiment of this review is positive, negative or neutral", + ... tokenizer_name_or_path="t5-base", + ... ) - Input Shape: (batch_size, total_virtual_tokens) + >>> # t5_model.shared is the word embeddings of the base model + >>> prompt_embedding = PromptEmbedding(config, t5_model.shared) + ``` - Output Shape: (batch_size, total_virtual_tokens, token_dim) + Input Shape: (`batch_size`, `total_virtual_tokens`) + + Output Shape: (`batch_size`, `total_virtual_tokens`, `token_dim`) """ def __init__(self, config, word_embeddings): diff --git a/src/peft/utils/config.py b/src/peft/utils/config.py index 2be3817..3ace67a 100644 --- a/src/peft/utils/config.py +++ b/src/peft/utils/config.py @@ -42,7 +42,7 @@ class TaskType(str, enum.Enum): class PeftConfigMixin(PushToHubMixin): r""" This is the base configuration class for PEFT adapter models. It contains all the methods that are common to all - PEFT adapter models. This class inherits from `transformers.utils.PushToHubMixin` which contains the methods to + PEFT adapter models. This class inherits from [`~transformers.utils.PushToHubMixin`] which contains the methods to push your model to the Hub. The method `save_pretrained` will save the configuration of your adapter model in a directory. The method `from_pretrained` will load the configuration of your adapter model from a directory. @@ -65,8 +65,8 @@ class PeftConfigMixin(PushToHubMixin): Args: save_directory (`str`): The directory where the configuration will be saved. - **kwargs: - Additional keyword arguments passed along to the `transformers.utils.PushToHubMixin.push_to_hub` + kwargs: + Additional keyword arguments passed along to the [`~transformers.utils.PushToHubMixin.push_to_hub`] method. """ if os.path.isfile(save_directory): @@ -88,8 +88,8 @@ class PeftConfigMixin(PushToHubMixin): Args: pretrained_model_name_or_path (`str`): - The directory or the hub-id where the configuration is saved. - **kwargs: + The directory or the Hub repository id where the configuration is saved. + kwargs: Additional keyword arguments passed along to the child class initialization. """ if os.path.isfile(os.path.join(pretrained_model_name_or_path, CONFIG_NAME)): @@ -128,7 +128,7 @@ class PeftConfigMixin(PushToHubMixin): @dataclass class PeftConfig(PeftConfigMixin): """ - This is the base configuration class to store the configuration of a :class:`~peft.PeftModel`. + This is the base configuration class to store the configuration of a [`PeftModel`]. Args: peft_type (Union[[`~peft.utils.config.PeftType`], `str`]): The type of Peft method to use. From 8e61e2637020d515f57d3d58ec6f52aba43cfa2a Mon Sep 17 00:00:00 2001 From: Steven Liu Date: Fri, 31 Mar 2023 14:41:14 -0700 Subject: [PATCH 21/30] fix kwargs --- src/peft/peft_model.py | 2 +- src/peft/utils/config.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py index 2b79f4c..0afd047 100644 --- a/src/peft/peft_model.py +++ b/src/peft/peft_model.py @@ -92,7 +92,7 @@ class PeftModel(PushToHubMixin, torch.nn.Module): save_directory (`str`): Directory where the adapter model and configuration files will be saved (will be created if it does not exist). - **kwargs: + kwargs (additional keyword arguments, *optional*): Additional keyword arguments passed along to the `push_to_hub` method. """ if os.path.isfile(save_directory): diff --git a/src/peft/utils/config.py b/src/peft/utils/config.py index 3ace67a..bdd7277 100644 --- a/src/peft/utils/config.py +++ b/src/peft/utils/config.py @@ -65,7 +65,7 @@ class PeftConfigMixin(PushToHubMixin): Args: save_directory (`str`): The directory where the configuration will be saved. - kwargs: + kwargs (additional keyword arguments, *optional*): Additional keyword arguments passed along to the [`~transformers.utils.PushToHubMixin.push_to_hub`] method. """ @@ -89,7 +89,7 @@ class PeftConfigMixin(PushToHubMixin): Args: pretrained_model_name_or_path (`str`): The directory or the Hub repository id where the configuration is saved. - kwargs: + kwargs (additional keyword arguments, *optional*): Additional keyword arguments passed along to the child class initialization. """ if os.path.isfile(os.path.join(pretrained_model_name_or_path, CONFIG_NAME)): @@ -145,8 +145,8 @@ class PeftConfig(PeftConfigMixin): @dataclass class PromptLearningConfig(PeftConfig): """ - This is the base configuration class to store the configuration of a Union[[`~peft.PrefixTuning`], - [`~peft.PromptEncoder`], [`~peft.PromptTuning`]]. + This is the base configuration class to store the configuration of [`PrefixTuning`], [`PromptEncoder`], or + [`PromptTuning`]. Args: num_virtual_tokens (`int`): The number of virtual tokens to use. From f948a9b4aecb149ac80c989a126703e9121a4bde Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 30 Mar 2023 10:47:03 -0700 Subject: [PATCH 22/30] build notebooks --- .github/workflows/build_documentation.yml | 3 ++- docs/source/_config.py | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 docs/source/_config.py diff --git a/.github/workflows/build_documentation.yml b/.github/workflows/build_documentation.yml index 309d35a..9e58cb8 100644 --- a/.github/workflows/build_documentation.yml +++ b/.github/workflows/build_documentation.yml @@ -13,5 +13,6 @@ jobs: with: commit_sha: ${{ github.sha }} package: peft + notebook_folder: peft_docs secrets: - token: ${{ secrets.HUGGINGFACE_PUSH }} + token: ${{ secrets.HUGGINGFACE_PUSH }} \ No newline at end of file diff --git a/docs/source/_config.py b/docs/source/_config.py new file mode 100644 index 0000000..a99c6a2 --- /dev/null +++ b/docs/source/_config.py @@ -0,0 +1,7 @@ +# docstyle-ignore +INSTALL_CONTENT = """ +# PEFT installation +! pip install peft accelerate transformers +# To install from source instead of the last release, comment the command above and uncomment the following one. +# ! pip install git+https://github.com/huggingface/peft.git +""" \ No newline at end of file From cfe992f0f9fe647fa2b0f011e8d6b7dc93d02ecb Mon Sep 17 00:00:00 2001 From: Steven Liu Date: Fri, 31 Mar 2023 16:54:12 -0700 Subject: [PATCH 23/30] make style --- docs/source/_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/_config.py b/docs/source/_config.py index a99c6a2..2974756 100644 --- a/docs/source/_config.py +++ b/docs/source/_config.py @@ -4,4 +4,4 @@ INSTALL_CONTENT = """ ! pip install peft accelerate transformers # To install from source instead of the last release, comment the command above and uncomment the following one. # ! pip install git+https://github.com/huggingface/peft.git -""" \ No newline at end of file +""" From e536616888d51b453ed354a6f1e243fecb02ea08 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Sat, 1 Apr 2023 14:54:46 +0200 Subject: [PATCH 24/30] [`core`] Fix offload issue (#248) * fix offload dir * remove offload index * safety checker * forward contrib credits from previous PR --------- Co-authored-by: cosimoiaia --- src/peft/peft_model.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py index 0afd047..f9573bb 100644 --- a/src/peft/peft_model.py +++ b/src/peft/peft_model.py @@ -164,6 +164,15 @@ class PeftModel(PushToHubMixin, torch.nn.Module): if getattr(model, "hf_device_map", None) is not None: device_map = kwargs.get("device_map", "auto") max_memory = kwargs.get("max_memory", None) + offload_dir = kwargs.get("offload_dir", None) + offload_index = kwargs.get("offload_index", None) + + dispatch_model_kwargs = {} + # Safety checker for previous `accelerate` versions + # `offload_index` was introduced in https://github.com/huggingface/accelerate/pull/873/ + if "offload_index" in inspect.signature(dispatch_model).parameters: + dispatch_model_kwargs["offload_index"] = offload_index + no_split_module_classes = model._no_split_modules if device_map != "sequential": max_memory = get_balanced_memory( @@ -176,7 +185,13 @@ class PeftModel(PushToHubMixin, torch.nn.Module): device_map = infer_auto_device_map( model, max_memory=max_memory, no_split_module_classes=no_split_module_classes ) - model = dispatch_model(model, device_map=device_map) + + model = dispatch_model( + model, + device_map=device_map, + offload_dir=offload_dir, + **dispatch_model_kwargs, + ) hook = AlignDevicesHook(io_same_device=True) if model.peft_config.peft_type == PeftType.LORA: add_hook_to_module(model.base_model.model, hook) From 7ef47be5f5f8d608773312d2f3e037f073f27e3b Mon Sep 17 00:00:00 2001 From: tpoisonooo Date: Mon, 3 Apr 2023 14:02:13 +0800 Subject: [PATCH 25/30] Update other.py typo --- src/peft/utils/other.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/peft/utils/other.py b/src/peft/utils/other.py index 132b033..585da64 100644 --- a/src/peft/utils/other.py +++ b/src/peft/utils/other.py @@ -34,7 +34,7 @@ def prepare_model_for_int8_training( model, output_embedding_layer_name="lm_head", use_gradient_checkpointing=True, layer_norm_names=["layer_norm"] ): r""" - This method wrapps the entire protocol for preparing a model before running a training. This includes: + This method wraps the entire protocol for preparing a model before running a training. This includes: 1- Cast the layernorm in fp32 2- making output embedding layer require grads 3- Add the upcasting of the lm head to fp32 From 39cbd7d8ed6b2fc56442ada66fba32898cfd00aa Mon Sep 17 00:00:00 2001 From: Guspan Tanadi <36249910+guspan-tanadi@users.noreply.github.com> Date: Mon, 3 Apr 2023 16:13:33 +0700 Subject: [PATCH 26/30] docs: have fix bit typo README Improve readability --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5653c1a..0dc7c8a 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Hardware: Single A100 80GB GPU with CPU RAM above 64GB | bigscience/bloomz-7b1 (7B params) | OOM GPU | 32GB GPU / 3.8GB CPU | 18.1GB GPU / 35GB CPU | Performance of PEFT-LoRA tuned [`bigscience/T0_3B`](https://huggingface.co/bigscience/T0_3B) on [`ought/raft/twitter_complaints`](https://huggingface.co/datasets/ought/raft/viewer/twitter_complaints) leaderboard. -A point to note is that we didn't try to sequeeze performance by playing around with input instruction templates, LoRA hyperparams and other training related hyperparams. Also, we didn't use the larger 13B [mt0-xxl](https://huggingface.co/bigscience/mt0-xxl) model. +A point to note is that we didn't try to squeeze performance by playing around with input instruction templates, LoRA hyperparams and other training related hyperparams. Also, we didn't use the larger 13B [mt0-xxl](https://huggingface.co/bigscience/mt0-xxl) model. So, we are already seeing comparable performance to SoTA with parameter efficient tuning. Also, the final checkpoint size is just `19MB` in comparison to `11GB` size of the backbone [`bigscience/T0_3B`](https://huggingface.co/bigscience/T0_3B) model. | Submission Name | Accuracy | @@ -81,7 +81,7 @@ GPU memory required by different settings during training is given below. The fi Hardware: Single A100 80GB GPU with CPU RAM above 64GB -| Model | Full Finetuning | PEFT-LoRA | PEFT-LoRA with Gradient Checkpoitning | +| Model | Full Finetuning | PEFT-LoRA | PEFT-LoRA with Gradient Checkpointing | | --------- | ---- | ---- | ---- | | CompVis/stable-diffusion-v1-4 | 27.5GB GPU / 3.97GB CPU | 15.5GB GPU / 3.84GB CPU | 8.12GB GPU / 3.77GB CPU | @@ -148,7 +148,7 @@ Another example is fine-tuning [`roberta-large`](https://huggingface.co/roberta- ## PEFT + 🤗 Accelerate -PEFT models work with 🤗 Accelerate out of the box. Use 🤗 Accelerate for Distributed training on various hardware such as GPUs, Apple Silicon devices etc during training. +PEFT models work with 🤗 Accelerate out of the box. Use 🤗 Accelerate for Distributed training on various hardware such as GPUs, Apple Silicon devices, etc during training. Use 🤗 Accelerate for inferencing on consumer hardware with small resources. ### Example of PEFT model training using 🤗 Accelerate's DeepSpeed integration From dd30335ffd32186fcf0ca1e10a569e87ca3690cf Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Mon, 3 Apr 2023 14:31:11 +0200 Subject: [PATCH 27/30] [`Automation`] Add stale bot (#247) * add stale bot * fix --- .github/workflows/stale.yml | 27 ++++++++++++++++ scripts/stale.py | 62 +++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 .github/workflows/stale.yml create mode 100644 scripts/stale.py diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000..8ad3e6d --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,27 @@ +name: Stale Bot + +on: + schedule: + - cron: "0 15 * * *" + +jobs: + close_stale_issues: + name: Close Stale Issues + if: github.repository == 'huggingface/peft' + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + + - name: Install requirements + run: | + pip install PyGithub + - name: Close stale issues + run: | + python scripts/stale.py \ No newline at end of file diff --git a/scripts/stale.py b/scripts/stale.py new file mode 100644 index 0000000..a0bd10a --- /dev/null +++ b/scripts/stale.py @@ -0,0 +1,62 @@ +# Copyright 2023 The HuggingFace Team, the AllenNLP library authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Script to close stale issue. Taken in part from the AllenNLP repository. +https://github.com/allenai/allennlp. +""" +from datetime import datetime as dt +import os + +from github import Github + + +LABELS_TO_EXEMPT = [ + "good first issue", + "good second issue", + "good difficult issue", + "feature request", + "new model", + "wip", +] + + +def main(): + g = Github(os.environ["GITHUB_TOKEN"]) + repo = g.get_repo("huggingface/peft") + open_issues = repo.get_issues(state="open") + + for issue in open_issues: + comments = sorted([comment for comment in issue.get_comments()], key=lambda i: i.created_at, reverse=True) + last_comment = comments[0] if len(comments) > 0 else None + if ( + last_comment is not None and last_comment.user.login == "github-actions[bot]" + and (dt.utcnow() - issue.updated_at).days > 7 + and (dt.utcnow() - issue.created_at).days >= 30 + and not any(label.name.lower() in LABELS_TO_EXEMPT for label in issue.get_labels()) + ): + issue.edit(state="closed") + elif ( + (dt.utcnow() - issue.updated_at).days > 23 + and (dt.utcnow() - issue.created_at).days >= 30 + and not any(label.name.lower() in LABELS_TO_EXEMPT for label in issue.get_labels()) + ): + issue.create_comment( + "This issue has been automatically marked as stale because it has not had " + "recent activity. If you think this still needs to be addressed " + "please comment on this thread.\n\n" + ) + + +if __name__ == "__main__": + main() \ No newline at end of file From 4ddb85ce1e2a25d11e5c32e485e2348df792da2c Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Mon, 3 Apr 2023 17:08:42 +0200 Subject: [PATCH 28/30] Update stale.py --- scripts/stale.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/stale.py b/scripts/stale.py index a0bd10a..e910135 100644 --- a/scripts/stale.py +++ b/scripts/stale.py @@ -28,6 +28,7 @@ LABELS_TO_EXEMPT = [ "feature request", "new model", "wip", + "PRs welcome to address this", ] @@ -59,4 +60,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() From 45d7aab39a0a580201209709cbc38d248cec0193 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 3 Apr 2023 08:51:01 -0700 Subject: [PATCH 29/30] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5653c1a..dffb656 100644 --- a/README.md +++ b/README.md @@ -25,10 +25,10 @@ Seamlessly integrated with 🤗 Accelerate for large scale models leveraging Dee Supported methods: -1. LoRA: [LORA: LOW-RANK ADAPTATION OF LARGE LANGUAGE MODELS](https://arxiv.org/pdf/2106.09685.pdf) +1. LoRA: [LORA: LOW-RANK ADAPTATION OF LARGE LANGUAGE MODELS](https://arxiv.org/abs/2106.09685) 2. Prefix Tuning: [Prefix-Tuning: Optimizing Continuous Prompts for Generation](https://aclanthology.org/2021.acl-long.353/), [P-Tuning v2: Prompt Tuning Can Be Comparable to Fine-tuning Universally Across Scales and Tasks](https://arxiv.org/pdf/2110.07602.pdf) -3. P-Tuning: [GPT Understands, Too](https://arxiv.org/pdf/2103.10385.pdf) -4. Prompt Tuning: [The Power of Scale for Parameter-Efficient Prompt Tuning](https://arxiv.org/pdf/2104.08691.pdf) +3. P-Tuning: [GPT Understands, Too](https://arxiv.org/abs/2103.10385) +4. Prompt Tuning: [The Power of Scale for Parameter-Efficient Prompt Tuning](https://arxiv.org/abs/2104.08691) ## Getting started From ff9a1edbfd2d405b86d50a2e5299cc1bbd49d887 Mon Sep 17 00:00:00 2001 From: toncho11 Date: Mon, 3 Apr 2023 18:28:11 +0200 Subject: [PATCH 30/30] Fixing a bug where a wrong parameter name is used. --- src/peft/peft_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py index f9573bb..85757b7 100644 --- a/src/peft/peft_model.py +++ b/src/peft/peft_model.py @@ -164,7 +164,7 @@ class PeftModel(PushToHubMixin, torch.nn.Module): if getattr(model, "hf_device_map", None) is not None: device_map = kwargs.get("device_map", "auto") max_memory = kwargs.get("max_memory", None) - offload_dir = kwargs.get("offload_dir", None) + offload_dir = kwargs.get("offload_folder", None) offload_index = kwargs.get("offload_index", None) dispatch_model_kwargs = {}