From 3d00af47994dbafd171dc007db8928da636a023a Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Fri, 24 Mar 2023 13:16:26 +0530 Subject: [PATCH 1/7] add docs --- .github/workflows/build_documentation.yml | 17 ++ .github/workflows/build_pr_documentation.yml | 16 + .github/workflows/delete_doc_comment.yml | 13 + Makefile | 6 +- docs/Makefile | 19 ++ docs/README.md | 267 +++++++++++++++++ docs/_toctree.yml | 16 + docs/index.mdx | 49 +++ docs/install.mdx | 46 +++ docs/package_reference/config | 0 docs/package_reference/peft_model | 0 docs/package_reference/tuners | 0 docs/quicktour.mdx | 300 +++++++++++++++++++ examples/lora_dreambooth/train_dreambooth.py | 4 +- 14 files changed, 749 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/build_documentation.yml create mode 100644 .github/workflows/build_pr_documentation.yml create mode 100644 .github/workflows/delete_doc_comment.yml create mode 100644 docs/Makefile create mode 100644 docs/README.md create mode 100644 docs/_toctree.yml create mode 100644 docs/index.mdx create mode 100644 docs/install.mdx create mode 100644 docs/package_reference/config create mode 100644 docs/package_reference/peft_model create mode 100644 docs/package_reference/tuners create mode 100644 docs/quicktour.mdx diff --git a/.github/workflows/build_documentation.yml b/.github/workflows/build_documentation.yml new file mode 100644 index 0000000..082ece2 --- /dev/null +++ b/.github/workflows/build_documentation.yml @@ -0,0 +1,17 @@ +name: Build documentation + +on: + push: + branches: + - main + - doc-builder* + - v*-release + +jobs: + build: + uses: huggingface/doc-builder/.github/workflows/build_main_documentation.yml@main + with: + commit_sha: ${{ github.sha }} + package: accelerate + secrets: + token: ${{ secrets.HUGGINGFACE_PUSH }} diff --git a/.github/workflows/build_pr_documentation.yml b/.github/workflows/build_pr_documentation.yml new file mode 100644 index 0000000..7506143 --- /dev/null +++ b/.github/workflows/build_pr_documentation.yml @@ -0,0 +1,16 @@ +name: Build PR Documentation + +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build: + uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@main + with: + commit_sha: ${{ github.event.pull_request.head.sha }} + pr_number: ${{ github.event.number }} + package: peft diff --git a/.github/workflows/delete_doc_comment.yml b/.github/workflows/delete_doc_comment.yml new file mode 100644 index 0000000..e86cc2d --- /dev/null +++ b/.github/workflows/delete_doc_comment.yml @@ -0,0 +1,13 @@ +name: Delete dev documentation + +on: + pull_request: + types: [ closed ] + + +jobs: + delete: + uses: huggingface/doc-builder/.github/workflows/delete_doc_comment.yml@main + with: + pr_number: ${{ github.event.number }} + package: peft diff --git a/Makefile b/Makefile index 61549db..3b6db1f 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: quality style test docs -check_dirs := src tests examples +check_dirs := src tests examples docs # Check that source code meets quality standards @@ -8,13 +8,13 @@ check_dirs := src tests examples quality: black --check $(check_dirs) ruff $(check_dirs) - doc-builder style src tests --max_len 119 --check_only + doc-builder style src tests docs --max_len 119 --check_only # Format source code automatically and check is there are any problems left that need manual fixing style: black $(check_dirs) ruff $(check_dirs) --fix - doc-builder style src tests --max_len 119 + doc-builder style src tests docs --max_len 119 test: pytest tests/ \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..8879933 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,19 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SOURCEDIR = source +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..32e51f1 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,267 @@ + + +# Generating the documentation + +To generate the documentation, you first have to build it. Several packages are necessary to build the doc, +you can install them with the following command, at the root of the code repository: + +```bash +pip install -e ".[docs]" +``` + +Then you need to install our special tool that builds the documentation: + +```bash +pip install git+https://github.com/huggingface/doc-builder +``` + +--- +**NOTE** + +You only need to generate the documentation to inspect it locally (if you're planning changes and want to +check how they look before committing for instance). You don't have to commit the built documentation. + +--- + +## Building the documentation + +Once you have setup the `doc-builder` and additional packages, you can generate the documentation by +typing the following command: + +```bash +doc-builder build accelerate docs/source/ --build_dir ~/tmp/test-build +``` + +You can adapt the `--build_dir` to set any temporary folder that you prefer. This command will create it and generate +the MDX files that will be rendered as the documentation on the main website. You can inspect them in your favorite +Markdown editor. + +## Previewing the documentation + +To preview the docs, first install the `watchdog` module with: + +```bash +pip install watchdog +``` + +Then run the following command: + +```bash +doc-builder preview {package_name} {path_to_docs} +``` + +For example: + +```bash +doc-builder preview transformers docs/source/en/ +``` + +The docs will be viewable at [http://localhost:3000](http://localhost:3000). You can also preview the docs once you have opened a PR. You will see a bot add a comment to a link where the documentation with your changes lives. + +--- +**NOTE** + +The `preview` command only works with existing doc files. When you add a completely new file, you need to update `_toctree.yml` & restart `preview` command (`ctrl-c` to stop it & call `doc-builder preview ...` again). + +--- + +## Adding a new element to the navigation bar + +Accepted files are Markdown (.md or .mdx). + +Create a file with its extension and put it in the source directory. You can then link it to the toc-tree by putting +the filename without the extension in the [`_toctree.yml`](https://github.com/huggingface/accelerate/blob/main/docs/source/_toctree.yml) file. + +## Renaming section headers and moving sections + +It helps to keep the old links working when renaming the section header and/or moving sections from one document to another. This is because the old links are likely to be used in Issues, Forums, and Social media and it'd make for a much more superior user experience if users reading those months later could still easily navigate to the originally intended information. + +Therefore, we simply keep a little map of moved sections at the end of the document where the original section was. The key is to preserve the original anchor. + +So if you renamed a section from: "Section A" to "Section B", then you can add at the end of the file: + +``` +Sections that were moved: + +[ Section A ] +``` +and of course, if you moved it to another file, then: + +``` +Sections that were moved: + +[ Section A ] +``` + +Use the relative style to link to the new file so that the versioned docs continue to work. + + +## Writing Documentation - Specification + +The `huggingface/accelerate` documentation follows the +[Google documentation](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) style for docstrings, +although we can write them directly in Markdown. + +### Adding a new tutorial + +Adding a new tutorial or section is done in two steps: + +- Add a new file under `./source`. This file can either be ReStructuredText (.rst) or Markdown (.md). +- Link that file in `./source/_toctree.yml` on the correct toc-tree. + +Make sure to put your new file under the proper section. It's unlikely to go in the first section (*Get Started*), so +depending on the intended targets (beginners, more advanced users, or researchers) it should go in sections two, three, or +four. + +### Writing source documentation + +Values that should be put in `code` should either be surrounded by backticks: \`like so\`. Note that argument names +and objects like True, None, or any strings should usually be put in `code`. + +When mentioning a class, function, or method, it is recommended to use our syntax for internal links so that our tool +adds a link to its documentation with this syntax: \[\`XXXClass\`\] or \[\`function\`\]. This requires the class or +function to be in the main package. + +If you want to create a link to some internal class or function, you need to +provide its path. For instance: \[\`utils.gather\`\]. This will be converted into a link with +`utils.gather` in the description. To get rid of the path and only keep the name of the object you are +linking to in the description, add a ~: \[\`~utils.gather\`\] will generate a link with `gather` in the description. + +The same works for methods so you can either use \[\`XXXClass.method\`\] or \[~\`XXXClass.method\`\]. + +#### Defining arguments in a method + +Arguments should be defined with the `Args:` (or `Arguments:` or `Parameters:`) prefix, followed by a line return and +an indentation. The argument should be followed by its type, with its shape if it is a tensor, a colon, and its +description: + +``` + Args: + n_layers (`int`): The number of layers of the model. +``` + +If the description is too long to fit in one line (more than 119 characters in total), another indentation is necessary +before writing the description after the argument. + +Finally, to maintain uniformity if any *one* description is too long to fit on one line, the +rest of the parameters should follow suit and have an indention before their description. + +Here's an example showcasing everything so far: + +``` + Args: + gradient_accumulation_steps (`int`, *optional*, default to 1): + The number of steps that should pass before gradients are accumulated. A number > 1 should be combined with `Accelerator.accumulate`. + cpu (`bool`, *optional*): + Whether or not to force the script to execute on CPU. Will ignore GPU available if set to `True` and force the execution on one process only. +``` + +For optional arguments or arguments with defaults we follow the following syntax: imagine we have a function with the +following signature: + +``` +def my_function(x: str = None, a: float = 1): +``` + +then its documentation should look like this: + +``` + Args: + x (`str`, *optional*): + This argument controls ... and has a description longer than 119 chars. + a (`float`, *optional*, defaults to 1): + This argument is used to ... and has a description longer than 119 chars. +``` + +Note that we always omit the "defaults to \`None\`" when None is the default for any argument. Also note that even +if the first line describing your argument type and its default gets long, you can't break it on several lines. You can +however write as many lines as you want in the indented description (see the example above with `input_ids`). + +#### Writing a multi-line code block + +Multi-line code blocks can be useful for displaying examples. They are done between two lines of three backticks as usual in Markdown: + + +```` +```python +# first line of code +# second line +# etc +``` +```` + +#### Writing a return block + +The return block should be introduced with the `Returns:` prefix, followed by a line return and an indentation. +The first line should be the type of the return, followed by a line return. No need to indent further for the elements +building the return. + +Here's an example of a single value return: + +``` + Returns: + `List[int]`: A list of integers in the range [0, 1] --- 1 for a special token, 0 for a sequence token. +``` + +Here's an example of a tuple return, comprising several objects: + +``` + Returns: + `tuple(torch.FloatTensor)` comprising various elements depending on the configuration ([`BertConfig`]) and inputs: + - ** loss** (*optional*, returned when `masked_lm_labels` is provided) `torch.FloatTensor` of shape `(1,)` -- + Total loss is the sum of the masked language modeling loss and the next sequence prediction (classification) loss. + - **prediction_scores** (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`) -- + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). +``` + +## Styling the docstring + +We have an automatic script running with the `make style` comment that will make sure that: +- the docstrings fully take advantage of the line width +- all code examples are formatted using black, like the code of the Transformers library + +This script may have some weird failures if you made a syntax mistake or if you uncover a bug. Therefore, it's +recommended to commit your changes before running `make style`, so you can revert the changes done by that script +easily. + +## Writing documentation examples + +The syntax for Example docstrings can look as follows: + +``` + Example: + + ```python + >>> import time + >>> from accelerate import Accelerator + >>> accelerator = Accelerator() + >>> if accelerator.is_main_process: + ... time.sleep(2) + >>> else: + ... print("I'm waiting for the main process to finish its sleep...") + >>> accelerator.wait_for_everyone() + >>> # Should print on every process at the same time + >>> print("Everyone is here") + ``` +``` + +The docstring should give a minimal, clear example of how the respective function +is to be used in inference and also include the expected (ideally sensible) +output. +Often, readers will try out the example before even going through the function +or class definitions. Therefore, it is of utmost importance that the example +works as expected. \ No newline at end of file diff --git a/docs/_toctree.yml b/docs/_toctree.yml new file mode 100644 index 0000000..4d6dd8b --- /dev/null +++ b/docs/_toctree.yml @@ -0,0 +1,16 @@ +- title: Get Started + sections: + - local: index + title: 🤗 PEFT + - local: quicktour + title: Quicktour + - local: installation + title: Installation +- title: Reference + sections: + - local: package_reference/peft_model + title: PEFT model + - local: package_reference/configs + title: Configuration + - local: package_reference/tuners + title: Tuners \ No newline at end of file diff --git a/docs/index.mdx b/docs/index.mdx new file mode 100644 index 0000000..9e1b4bd --- /dev/null +++ b/docs/index.mdx @@ -0,0 +1,49 @@ + + +# PEFT + +🤗 PEFT is a library that enables using State-of-the-art Parameter-Efficient Fine-Tuning (PEFT) methods. + +PEFT methods enable efficient adaptation of pre-trained language models (PLMs) to +various downstream applications without fine-tuning all the model's parameters. +Fine-tuning large-scale PLMs is often prohibitively costly. +In this regard, PEFT methods only fine-tune a small number of (extra) model parameters, +thereby greatly decreasing the computational and storage costs. +Recent State-of-the-Art PEFT techniques achieve performance comparable to that of full fine-tuning. + +Seamlessly integrated with 🤗 Accelerate for large scale models leveraging DeepSpeed and Big Model Inference. + +Supported methods, with more coming soon: + +1. LoRA: [LORA: LOW-RANK ADAPTATION OF LARGE LANGUAGE MODELS](https://arxiv.org/pdf/2106.09685.pdf) +2. Prefix Tuning: [Prefix-Tuning: Optimizing Continuous Prompts for Generation](https://aclanthology.org/2021.acl-long.353/), [P-Tuning v2: Prompt Tuning Can Be Comparable to Fine-tuning Universally Across Scales and Tasks](https://arxiv.org/pdf/2110.07602.pdf) +3. P-Tuning: [GPT Understands, Too](https://arxiv.org/pdf/2103.10385.pdf) +4. Prompt Tuning: [The Power of Scale for Parameter-Efficient Prompt Tuning](https://arxiv.org/pdf/2104.08691.pdf) + +## Getting started + +```python +from transformers import AutoModelForSeq2SeqLM +from peft import get_peft_config, get_peft_model, LoraConfig, TaskType + +model_name_or_path = "bigscience/mt0-large" +tokenizer_name_or_path = "bigscience/mt0-large" + +peft_config = LoraConfig(task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1) + +model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path) +model = get_peft_model(model, peft_config) +model.print_trainable_parameters() +# output: trainable params: 2359296 || all params: 1231940608 || trainable%: 0.19151053100118282 +``` + diff --git a/docs/install.mdx b/docs/install.mdx new file mode 100644 index 0000000..e086ed8 --- /dev/null +++ b/docs/install.mdx @@ -0,0 +1,46 @@ + + +# Installation and Configuration + +Before you start, you will need to setup your environment, install the appropriate packages, and configure 🤗 PEFT. 🤗 PEFT is tested on **Python 3.7+**. + +## Installing 🤗 PEFT + +🤗 PEFT is available on pypi, as well as on GitHub. Details to install from each are below: + +### pip + +To install 🤗 PEFT from pypi, perform: + +```bash +pip install peft +``` + +### Source + +New features are added every day that haven't been released yet. To try them out yourself, install +from the GitHub repository: + +```bash +pip install git+https://github.com/huggingface/peft +``` + +If you're working on contributing to the library or wish to play with the source code and see live +results as you run the code, an editable version can be installed from a locally-cloned version of the +repository: + +```bash +git clone https://github.com/huggingface/peft +cd peft +pip install -e . +``` diff --git a/docs/package_reference/config b/docs/package_reference/config new file mode 100644 index 0000000..e69de29 diff --git a/docs/package_reference/peft_model b/docs/package_reference/peft_model new file mode 100644 index 0000000..e69de29 diff --git a/docs/package_reference/tuners b/docs/package_reference/tuners new file mode 100644 index 0000000..e69de29 diff --git a/docs/quicktour.mdx b/docs/quicktour.mdx new file mode 100644 index 0000000..45ee22c --- /dev/null +++ b/docs/quicktour.mdx @@ -0,0 +1,300 @@ + + +# Quick tour + +Let's have a look at the 🤗 PEFT main features and traps to avoid. + +## Main use + +To use 🤗 PEFT in your script, you have to follow below steps: + +1. Create a `PeftConfig` object corresponding to your PEFT method. +Please refer to the [Config Page](package_reference/config) for more details. +Below, we will use `LoRAConfig` for demonstration. + +```python +from peft import LoraConfig, TaskType + +peft_config = LoraConfig(task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1) +``` + +Here, `task_type` is the type of task you are training your model for. +For available task types, please refer [TaskType](package_reference/config#peft.config.TaskType). + +2. Load the base model you want to fine-tune. + +```python +from transformers import AutoModelForSeq2SeqLM + +model_name_or_path = "bigscience/mt0-large" +tokenizer_name_or_path = "bigscience/mt0-large" +model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path) +``` + +3. Preprocess your model if you use `bitsandbytes` for INT-8 quantized training; else skip this step. + +```python +from peft import prepare_model_for_int8_training + +model = prepare_model_for_int8_training(model) +``` + +4. Wrap your model in the `PeftModel` object using the `get_peft_model` function. Also, check the number of trainable parameters of your model. + +```python +from peft import get_peft_model + +model = get_peft_model(model, peft_config) +model.print_trainable_parameters() +# output: trainable params: 2359296 || all params: 1231940608 || trainable%: 0.19151053100118282 +``` + +5. Voila 🎉. Now, train the model using 🤗 Transformers Trainer API, 🤗 Accelerate or any custom PyTroch training loop. +Please refer example [peft_lora_seq2seq.ipynb](https://github.com/huggingface/peft/blob/main/examples/conditional_generation/peft_lora_seq2seq.ipynb) for an end-to-end example. + +### Saving/loading a model + +1. Save your model using the `save_pretrained` function. + +```python +model.save_pretrained("output_dir") +# model.push_to_hub("my_awesome_peft_model") also works +``` + +This will only save the incremental PEFT weights that were trained. +For example, you can find the `bigscience/T0_3B` tuned using LoRA on the `twitter_complaints` raft dataset here: +[smangrul/twitter_complaints_bigscience_T0_3B_LORA_SEQ_2_SEQ_LM](https://huggingface.co/smangrul/twitter_complaints_bigscience_T0_3B_LORA_SEQ_2_SEQ_LM). +Notice that it only contains 2 files: `adapter_config.json` and `adapter_model.bin` with the latter being just 19MB. + +2. Load your model using the `from_pretrained` function. + +```diff + from transformers import AutoModelForSeq2SeqLM ++ from peft import PeftModel, PeftConfig + ++ peft_model_id = "smangrul/twitter_complaints_bigscience_T0_3B_LORA_SEQ_2_SEQ_LM" ++ config = PeftConfig.from_pretrained(peft_model_id) + model = AutoModelForSeq2SeqLM.from_pretrained(config.base_model_name_or_path) ++ model = PeftModel.from_pretrained(model, peft_model_id) + tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path) + + model = model.to(device) + model.eval() + inputs = tokenizer("Tweet text : @HondaCustSvc Your customer service has been horrible during the recall process. I will never purchase a Honda again. Label :", return_tensors="pt") + + with torch.no_grad(): + outputs = model.generate(input_ids=inputs["input_ids"].to("cuda"), max_new_tokens=10) + print(tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True)[0]) +# 'complaint' +``` + +## Launching your distributed script + +PEFT models work with 🤗 Accelerate out of the box. +Use 🤗 Accelerate for Distributed training on various hardware such as GPUs, Apple Silicon devices etc during training. +Use 🤗 Accelerate for inferencing on consumer hardware with small resources. + +### Example of PEFT model training using 🤗 Accelerate's DeepSpeed integration + +DeepSpeed version required `v0.8.0`. An example is provided in `~examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py`. + a. First, run `accelerate config --config_file ds_zero3_cpu.yaml` and answer the questionnaire. + Below are the contents of the config file. + ```yaml + compute_environment: LOCAL_MACHINE + deepspeed_config: + gradient_accumulation_steps: 1 + gradient_clipping: 1.0 + offload_optimizer_device: cpu + offload_param_device: cpu + zero3_init_flag: true + zero3_save_16bit_model: true + zero_stage: 3 + distributed_type: DEEPSPEED + downcast_bf16: 'no' + dynamo_backend: 'NO' + fsdp_config: {} + machine_rank: 0 + main_training_function: main + megatron_lm_config: {} + mixed_precision: 'no' + num_machines: 1 + num_processes: 1 + rdzv_backend: static + same_network: true + use_cpu: false + ``` + b. run the below command to launch the example script + ```bash + accelerate launch --config_file ds_zero3_cpu.yaml examples/peft_lora_seq2seq_accelerate_ds_zero3_offload.py + ``` + + c. output logs: + ```bash + GPU Memory before entering the train : 1916 + GPU Memory consumed at the end of the train (end-begin): 66 + GPU Peak Memory consumed during the train (max-begin): 7488 + GPU Total Peak Memory consumed during the train (max): 9404 + CPU Memory before entering the train : 19411 + CPU Memory consumed at the end of the train (end-begin): 0 + CPU Peak Memory consumed during the train (max-begin): 0 + CPU Total Peak Memory consumed during the train (max): 19411 + epoch=4: train_ppl=tensor(1.0705, device='cuda:0') train_epoch_loss=tensor(0.0681, device='cuda:0') + 100%|████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:27<00:00, 3.92s/it] + GPU Memory before entering the eval : 1982 + GPU Memory consumed at the end of the eval (end-begin): -66 + GPU Peak Memory consumed during the eval (max-begin): 672 + GPU Total Peak Memory consumed during the eval (max): 2654 + CPU Memory before entering the eval : 19411 + CPU Memory consumed at the end of the eval (end-begin): 0 + CPU Peak Memory consumed during the eval (max-begin): 0 + CPU Total Peak Memory consumed during the eval (max): 19411 + accuracy=100.0 + eval_preds[:10]=['no complaint', 'no complaint', 'complaint', 'complaint', 'no complaint', 'no complaint', 'no complaint', 'complaint', 'complaint', 'no complaint'] + dataset['train'][label_column][:10]=['no complaint', 'no complaint', 'complaint', 'complaint', 'no complaint', 'no complaint', 'no complaint', 'complaint', 'complaint', 'no complaint'] + ``` + +### Example of PEFT model inference using 🤗 Accelerate's Big Model Inferencing capabilities +An example is provided in `~examples/causal_language_modeling/peft_lora_clm_accelerate_big_model_inference.ipynb`. + +## Model Support matrix + +### Causal Language Modeling +| Model | LoRA | Prefix Tuning | P-Tuning | Prompt Tuning | +|--------------| ---- | ---- | ---- | ---- | +| GPT-2 | ✅ | ✅ | ✅ | ✅ | +| Bloom | ✅ | ✅ | ✅ | ✅ | +| OPT | ✅ | ✅ | ✅ | ✅ | +| GPT-Neo | ✅ | ✅ | ✅ | ✅ | +| GPT-J | ✅ | ✅ | ✅ | ✅ | +| GPT-NeoX-20B | ✅ | ✅ | ✅ | ✅ | +| LLaMA | ✅ | ✅ | ✅ | ✅ | +| ChatGLM | ✅ | ✅ | ✅ | ✅ | + +### Conditional Generation +| Model | LoRA | Prefix Tuning | P-Tuning | Prompt Tuning | +| --------- | ---- | ---- | ---- | ---- | +| T5 | ✅ | ✅ | ✅ | ✅ | +| BART | ✅ | ✅ | ✅ | ✅ | + +### Sequence Classification +| Model | LoRA | Prefix Tuning | P-Tuning | Prompt Tuning | +| --------- | ---- | ---- | ---- | ---- | +| BERT | ✅ | ✅ | ✅ | ✅ | +| RoBERTa | ✅ | ✅ | ✅ | ✅ | +| GPT-2 | ✅ | ✅ | ✅ | ✅ | +| Bloom | ✅ | ✅ | ✅ | ✅ | +| OPT | ✅ | ✅ | ✅ | ✅ | +| GPT-Neo | ✅ | ✅ | ✅ | ✅ | +| GPT-J | ✅ | ✅ | ✅ | ✅ | +| Deberta | ✅ | | ✅ | ✅ | +| Deberta-v2 | ✅ | | ✅ | ✅ | + +### Token Classification +| Model | LoRA | Prefix Tuning | P-Tuning | Prompt Tuning | +| --------- | ---- | ---- | ---- | ---- | +| BERT | ✅ | ✅ | | | +| RoBERTa | ✅ | ✅ | | | +| GPT-2 | ✅ | ✅ | | | +| Bloom | ✅ | ✅ | | | +| OPT | ✅ | ✅ | | | +| GPT-Neo | ✅ | ✅ | | | +| GPT-J | ✅ | ✅ | | | +| Deberta | ✅ | | | | +| Deberta-v2 | ✅ | | | | + +### Text-to-Image Generation + +| Model | LoRA | Prefix Tuning | P-Tuning | Prompt Tuning | +| --------- | ---- | ---- | ---- | ---- | +| Stable Diffusion | ✅ | | | | + + +### Image Classification + +| Model | LoRA | Prefix Tuning | P-Tuning | Prompt Tuning | +| --------- | ---- | ---- | ---- | ---- | +| ViT | ✅ | | | | +| Swin | ✅ | | | | + +___Note that we have tested LoRA for [ViT](https://huggingface.co/docs/transformers/model_doc/vit) and [Swin](https://huggingface.co/docs/transformers/model_doc/swin) for fine-tuning on image classification. However, it should be possible to use LoRA for any compatible model [provided](https://huggingface.co/models?pipeline_tag=image-classification&sort=downloads&search=vit) by 🤗 Transformers. Check out the respective +examples to learn more. If you run into problems, please open an issue.___ + +The same principle applies to our [segmentation models](https://huggingface.co/models?pipeline_tag=image-segmentation&sort=downloads) as well. + +### Semantic Segmentation + +| Model | LoRA | Prefix Tuning | P-Tuning | Prompt Tuning | +| --------- | ---- | ---- | ---- | ---- | +| SegFormer | ✅ | | | | + + +## Other caveats + +1. Below is an example of using PyTorch FSDP for training. However, it doesn't lead to +any GPU memory savings. Please refer to issue [[FSDP] FSDP with CPU offload consumes 1.65X more GPU memory when training models with most of the params frozen](https://github.com/pytorch/pytorch/issues/91165). + + ```python + from peft.utils.other import fsdp_auto_wrap_policy + + + if os.environ.get("ACCELERATE_USE_FSDP", None) is not None: + accelerator.state.fsdp_plugin.auto_wrap_policy = fsdp_auto_wrap_policy(model) + + model = accelerator.prepare(model) + ``` + + Example of parameter efficient tuning with [`mt0-xxl`](https://huggingface.co/bigscience/mt0-xxl) base model using 🤗 Accelerate is provided in `~examples/conditional_generation/peft_lora_seq2seq_accelerate_fsdp.py`. + a. First, run `accelerate config --config_file fsdp_config.yaml` and answer the questionnaire. + Below are the contents of the config file. + ```yaml + command_file: null + commands: null + compute_environment: LOCAL_MACHINE + deepspeed_config: {} + distributed_type: FSDP + downcast_bf16: 'no' + dynamo_backend: 'NO' + fsdp_config: + fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP + fsdp_backward_prefetch_policy: BACKWARD_PRE + fsdp_offload_params: true + fsdp_sharding_strategy: 1 + fsdp_state_dict_type: FULL_STATE_DICT + fsdp_transformer_layer_cls_to_wrap: T5Block + gpu_ids: null + machine_rank: 0 + main_process_ip: null + main_process_port: null + main_training_function: main + megatron_lm_config: {} + mixed_precision: 'no' + num_machines: 1 + num_processes: 2 + rdzv_backend: static + same_network: true + tpu_name: null + tpu_zone: null + use_cpu: false + ``` + b. run the below command to launch the example script + ```bash + accelerate launch --config_file fsdp_config.yaml examples/peft_lora_seq2seq_accelerate_fsdp.py + ``` + +2. When using `P_TUNING` or `PROMPT_TUNING` with `SEQ_2_SEQ` task, remember to remove the `num_virtual_token` virtual prompt predictions from the left side of the model outputs during evaluations. + +3. For encoder-decoder models, `P_TUNING` or `PROMPT_TUNING` doesn't support the `generate` functionality of transformers because `generate` strictly requires `decoder_input_ids` but +`P_TUNING`/`PROMPT_TUNING` append soft prompt embeddings to `input_embeds` to create +new `input_embeds` to be given to the model. Therefore, `generate` doesn't support this yet. + +4. When using ZeRO3 with zero3_init_flag=True, if you find the GPU memory increase with training steps. we might need to set zero3_init_flag=false in accelerate config.yaml. The related issue is [[BUG] memory leak under zero.Init](https://github.com/microsoft/DeepSpeed/issues/2637) \ No newline at end of file diff --git a/examples/lora_dreambooth/train_dreambooth.py b/examples/lora_dreambooth/train_dreambooth.py index 9145eca..32f78a8 100644 --- a/examples/lora_dreambooth/train_dreambooth.py +++ b/examples/lora_dreambooth/train_dreambooth.py @@ -1063,7 +1063,9 @@ def main(args): ) text_encoder_state_dict = {f"text_encoder_{k}": v for k, v in text_encoder_state_dict.items()} state_dict.update(text_encoder_state_dict) - lora_config["text_encoder_peft_config"] = unwarpped_text_encoder.get_peft_config_as_dict(inference=True) + lora_config["text_encoder_peft_config"] = unwarpped_text_encoder.get_peft_config_as_dict( + inference=True + ) accelerator.print(state_dict) accelerator.save(state_dict, os.path.join(args.output_dir, f"{args.instance_prompt}_lora.pt")) From 13476a807ccd86189809dd00e627da93dfab5aff Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Mon, 27 Mar 2023 13:44:00 +0530 Subject: [PATCH 2/7] Apply suggestions from code review Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- .github/workflows/build_documentation.yml | 2 +- Makefile | 4 +-- docs/README.md | 8 ++--- docs/_toctree.yml | 2 +- docs/index.mdx | 15 +++------ docs/install.mdx | 15 ++++----- docs/quicktour.mdx | 41 +++++++++++------------ 7 files changed, 39 insertions(+), 48 deletions(-) diff --git a/.github/workflows/build_documentation.yml b/.github/workflows/build_documentation.yml index 082ece2..309d35a 100644 --- a/.github/workflows/build_documentation.yml +++ b/.github/workflows/build_documentation.yml @@ -12,6 +12,6 @@ jobs: uses: huggingface/doc-builder/.github/workflows/build_main_documentation.yml@main with: commit_sha: ${{ github.sha }} - package: accelerate + package: peft secrets: token: ${{ secrets.HUGGINGFACE_PUSH }} diff --git a/Makefile b/Makefile index 3b6db1f..145a375 100644 --- a/Makefile +++ b/Makefile @@ -8,13 +8,13 @@ check_dirs := src tests examples docs quality: black --check $(check_dirs) ruff $(check_dirs) - doc-builder style src tests docs --max_len 119 --check_only + doc-builder style src/peft tests docs/source --max_len 119 --check_only # Format source code automatically and check is there are any problems left that need manual fixing style: black $(check_dirs) ruff $(check_dirs) --fix - doc-builder style src tests docs --max_len 119 + doc-builder style src/peft tests docs/source --max_len 119 test: pytest tests/ \ No newline at end of file diff --git a/docs/README.md b/docs/README.md index 32e51f1..5955736 100644 --- a/docs/README.md +++ b/docs/README.md @@ -43,7 +43,7 @@ Once you have setup the `doc-builder` and additional packages, you can generate typing the following command: ```bash -doc-builder build accelerate docs/source/ --build_dir ~/tmp/test-build +doc-builder build peft docs/source/ --build_dir ~/tmp/test-build ``` You can adapt the `--build_dir` to set any temporary folder that you prefer. This command will create it and generate @@ -67,7 +67,7 @@ doc-builder preview {package_name} {path_to_docs} For example: ```bash -doc-builder preview transformers docs/source/en/ +doc-builder preview peft docs/source ``` The docs will be viewable at [http://localhost:3000](http://localhost:3000). You can also preview the docs once you have opened a PR. You will see a bot add a comment to a link where the documentation with your changes lives. @@ -84,7 +84,7 @@ The `preview` command only works with existing doc files. When you add a complet Accepted files are Markdown (.md or .mdx). Create a file with its extension and put it in the source directory. You can then link it to the toc-tree by putting -the filename without the extension in the [`_toctree.yml`](https://github.com/huggingface/accelerate/blob/main/docs/source/_toctree.yml) file. +the filename without the extension in the [`_toctree.yml`](https://github.com/huggingface/peft/blob/main/docs/source/_toctree.yml) file. ## Renaming section headers and moving sections @@ -112,7 +112,7 @@ Use the relative style to link to the new file so that the versioned docs contin ## Writing Documentation - Specification -The `huggingface/accelerate` documentation follows the +The `huggingface/peft` documentation follows the [Google documentation](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) style for docstrings, although we can write them directly in Markdown. diff --git a/docs/_toctree.yml b/docs/_toctree.yml index 4d6dd8b..211b83f 100644 --- a/docs/_toctree.yml +++ b/docs/_toctree.yml @@ -1,7 +1,7 @@ - title: Get Started sections: - local: index - title: 🤗 PEFT + title: 🤗 PEFT - local: quicktour title: Quicktour - local: installation diff --git a/docs/index.mdx b/docs/index.mdx index 9e1b4bd..4f5776f 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -12,18 +12,13 @@ specific language governing permissions and limitations under the License. # PEFT -🤗 PEFT is a library that enables using State-of-the-art Parameter-Efficient Fine-Tuning (PEFT) methods. +🤗 PEFT, or Parameter-Efficient Fine-Tuning (PEFT), is a library for efficiently adapting pre-trained language models (PLMs) to various downstream applications without fine-tuning all the model's parameters. +PEFT methods only fine-tune a small number of (extra) model parameters, significantly decreasing computational and storage costs because fine-tuning large-scale PLMs is prohibitively costly. +Recent state-of-the-art PEFT techniques achieve performance comparable to that of full fine-tuning. -PEFT methods enable efficient adaptation of pre-trained language models (PLMs) to -various downstream applications without fine-tuning all the model's parameters. -Fine-tuning large-scale PLMs is often prohibitively costly. -In this regard, PEFT methods only fine-tune a small number of (extra) model parameters, -thereby greatly decreasing the computational and storage costs. -Recent State-of-the-Art PEFT techniques achieve performance comparable to that of full fine-tuning. +PEFT is seamlessly integrated with 🤗 Accelerate for large-scale models leveraging DeepSpeed and [Big Model Inference](https://huggingface.co/docs/accelerate/usage_guides/big_modeling). -Seamlessly integrated with 🤗 Accelerate for large scale models leveraging DeepSpeed and Big Model Inference. - -Supported methods, with more coming soon: +Supported methods include: 1. LoRA: [LORA: LOW-RANK ADAPTATION OF LARGE LANGUAGE MODELS](https://arxiv.org/pdf/2106.09685.pdf) 2. Prefix Tuning: [Prefix-Tuning: Optimizing Continuous Prompts for Generation](https://aclanthology.org/2021.acl-long.353/), [P-Tuning v2: Prompt Tuning Can Be Comparable to Fine-tuning Universally Across Scales and Tasks](https://arxiv.org/pdf/2110.07602.pdf) diff --git a/docs/install.mdx b/docs/install.mdx index e086ed8..5f5ecff 100644 --- a/docs/install.mdx +++ b/docs/install.mdx @@ -10,26 +10,23 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express o specific language governing permissions and limitations under the License. --> -# Installation and Configuration +# Installation Before you start, you will need to setup your environment, install the appropriate packages, and configure 🤗 PEFT. 🤗 PEFT is tested on **Python 3.7+**. -## Installing 🤗 PEFT +🤗 PEFT is available on pypi, as well as GitHub: -🤗 PEFT is available on pypi, as well as on GitHub. Details to install from each are below: +## pip -### pip - -To install 🤗 PEFT from pypi, perform: +To install 🤗 PEFT from pypi: ```bash pip install peft ``` -### Source +## Source -New features are added every day that haven't been released yet. To try them out yourself, install -from the GitHub repository: +New features that haven't been released yet are added every day, which also means there may be some bugs. To try them out, install from the GitHub repository: ```bash pip install git+https://github.com/huggingface/peft diff --git a/docs/quicktour.mdx b/docs/quicktour.mdx index 45ee22c..e0eb37f 100644 --- a/docs/quicktour.mdx +++ b/docs/quicktour.mdx @@ -12,15 +12,16 @@ specific language governing permissions and limitations under the License. # Quick tour -Let's have a look at the 🤗 PEFT main features and traps to avoid. +Let's have a look at 🤗 PEFT's main features and learn how to set up a `PeftModel` and train it with 🤗 Accelerate's DeepSpeed integration and use it for inference. ## Main use -To use 🤗 PEFT in your script, you have to follow below steps: +To use 🤗 PEFT in your script: -1. Create a `PeftConfig` object corresponding to your PEFT method. -Please refer to the [Config Page](package_reference/config) for more details. -Below, we will use `LoRAConfig` for demonstration. +1. Each PEFT method is defined by a `PeftConfig` object. + +Create a `PeftConfig` object corresponding to your PEFT method (see the [Configuration](package_reference/config) reference for more details) and [`TaskType`], the type of task you're training your model for. +This example trains the [`bigscience/mt0-large`](https://huggingface.co/bigscience/mt0-large) model with the Low-Rank Adaptation of Large Language Models (LoRA) method. Load the `LoRAConfig`, and specify the `task_type` for sequence-to-sequence language modeling. ```python from peft import LoraConfig, TaskType @@ -41,7 +42,7 @@ tokenizer_name_or_path = "bigscience/mt0-large" model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path) ``` -3. Preprocess your model if you use `bitsandbytes` for INT-8 quantized training; else skip this step. +3. Preprocess your model if you use [`bitsandbytes`](https://github.com/TimDettmers/bitsandbytes) for `int8` quantized training; otherwise, skip this step. ```python from peft import prepare_model_for_int8_training @@ -59,8 +60,7 @@ model.print_trainable_parameters() # output: trainable params: 2359296 || all params: 1231940608 || trainable%: 0.19151053100118282 ``` -5. Voila 🎉. Now, train the model using 🤗 Transformers Trainer API, 🤗 Accelerate or any custom PyTroch training loop. -Please refer example [peft_lora_seq2seq.ipynb](https://github.com/huggingface/peft/blob/main/examples/conditional_generation/peft_lora_seq2seq.ipynb) for an end-to-end example. +5. Voila 🎉! Now, train the model using the 🤗 Transformers Trainer API, 🤗 Accelerate, or any custom PyTroch training loop (take a look at the end-to-end [example](https://github.com/huggingface/peft/blob/main/examples/conditional_generation/peft_lora_seq2seq.ipynb) of training [`bigscience/mt0-large`](https://huggingface.co/bigscience/mt0-large)). ### Saving/loading a model @@ -71,10 +71,9 @@ model.save_pretrained("output_dir") # model.push_to_hub("my_awesome_peft_model") also works ``` -This will only save the incremental PEFT weights that were trained. -For example, you can find the `bigscience/T0_3B` tuned using LoRA on the `twitter_complaints` raft dataset here: -[smangrul/twitter_complaints_bigscience_T0_3B_LORA_SEQ_2_SEQ_LM](https://huggingface.co/smangrul/twitter_complaints_bigscience_T0_3B_LORA_SEQ_2_SEQ_LM). -Notice that it only contains 2 files: `adapter_config.json` and `adapter_model.bin` with the latter being just 19MB. +This only saves the incremental PEFT weights that were trained. +For example, [smangrul/twitter_complaints_bigscience_T0_3B_LORA_SEQ_2_SEQ_LM](https://huggingface.co/smangrul/twitter_complaints_bigscience_T0_3B_LORA_SEQ_2_SEQ_LM) is a `bigscience/T0_3B`model finetuned with LoRA on the [`twitter_complaints`](https://huggingface.co/datasets/ought/raft/viewer/twitter_complaints/train) RAFT dataset. +Notice that it only contains 2 files: `adapter_config.json` and `adapter_model.bin`, with the latter being just 19MB. 2. Load your model using the `from_pretrained` function. @@ -101,14 +100,14 @@ Notice that it only contains 2 files: `adapter_config.json` and `adapter_model.b ## Launching your distributed script PEFT models work with 🤗 Accelerate out of the box. -Use 🤗 Accelerate for Distributed training on various hardware such as GPUs, Apple Silicon devices etc during training. -Use 🤗 Accelerate for inferencing on consumer hardware with small resources. +You can use 🤗 Accelerate for distributed training on various hardware such as GPUs, or Apple Silicon devices during training, and for inference on consumer hardware with fewer resources. -### Example of PEFT model training using 🤗 Accelerate's DeepSpeed integration +### Train with 🤗 Accelerate's DeepSpeed integration -DeepSpeed version required `v0.8.0`. An example is provided in `~examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py`. - a. First, run `accelerate config --config_file ds_zero3_cpu.yaml` and answer the questionnaire. - Below are the contents of the config file. +You'll need DeepSpeed version `v0.8.0` for this example. Feel free to check out the full example [script](https://github.com/huggingface/peft/blob/main/examples/conditional_generation/peft_lora_seq2seq_accelerate_ds_zero3_offload.py) for more details! + +1. Run `accelerate config --config_file ds_zero3_cpu.yaml` and answer the questionnaire to setup your environment. +Below are the contents of the config file. ```yaml compute_environment: LOCAL_MACHINE deepspeed_config: @@ -133,12 +132,12 @@ DeepSpeed version required `v0.8.0`. An example is provided in `~examples/condit same_network: true use_cpu: false ``` - b. run the below command to launch the example script +2. Run the following command to launch the example script: ```bash accelerate launch --config_file ds_zero3_cpu.yaml examples/peft_lora_seq2seq_accelerate_ds_zero3_offload.py ``` - c. output logs: +You'll see some output logs that look like this: ```bash GPU Memory before entering the train : 1916 GPU Memory consumed at the end of the train (end-begin): 66 @@ -163,7 +162,7 @@ DeepSpeed version required `v0.8.0`. An example is provided in `~examples/condit dataset['train'][label_column][:10]=['no complaint', 'no complaint', 'complaint', 'complaint', 'no complaint', 'no complaint', 'no complaint', 'complaint', 'complaint', 'no complaint'] ``` -### Example of PEFT model inference using 🤗 Accelerate's Big Model Inferencing capabilities +### Inference with 🤗 Accelerate's Big Model Inference An example is provided in `~examples/causal_language_modeling/peft_lora_clm_accelerate_big_model_inference.ipynb`. ## Model Support matrix From 891584c8d93dc0c9aeff578be5f656fb25d43745 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 28 Mar 2023 13:55:43 +0000 Subject: [PATCH 3/7] fix ci dreambooth --- examples/lora_dreambooth/train_dreambooth.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/lora_dreambooth/train_dreambooth.py b/examples/lora_dreambooth/train_dreambooth.py index 9145eca..32f78a8 100644 --- a/examples/lora_dreambooth/train_dreambooth.py +++ b/examples/lora_dreambooth/train_dreambooth.py @@ -1063,7 +1063,9 @@ def main(args): ) text_encoder_state_dict = {f"text_encoder_{k}": v for k, v in text_encoder_state_dict.items()} state_dict.update(text_encoder_state_dict) - lora_config["text_encoder_peft_config"] = unwarpped_text_encoder.get_peft_config_as_dict(inference=True) + lora_config["text_encoder_peft_config"] = unwarpped_text_encoder.get_peft_config_as_dict( + inference=True + ) accelerator.print(state_dict) accelerator.save(state_dict, os.path.join(args.output_dir, f"{args.instance_prompt}_lora.pt")) From 4626b36e273470884041f4ca84f44930c16c59f3 Mon Sep 17 00:00:00 2001 From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com> Date: Wed, 29 Mar 2023 17:12:32 +0530 Subject: [PATCH 4/7] addressing remaining comments --- docs/{ => source}/_toctree.yml | 0 docs/{ => source}/index.mdx | 18 ------------------ docs/{ => source}/install.mdx | 0 docs/{ => source}/package_reference/config | 0 docs/{ => source}/package_reference/peft_model | 0 docs/{ => source}/package_reference/tuners | 0 docs/{ => source}/quicktour.mdx | 3 --- 7 files changed, 21 deletions(-) rename docs/{ => source}/_toctree.yml (100%) rename docs/{ => source}/index.mdx (75%) rename docs/{ => source}/install.mdx (100%) rename docs/{ => source}/package_reference/config (100%) rename docs/{ => source}/package_reference/peft_model (100%) rename docs/{ => source}/package_reference/tuners (100%) rename docs/{ => source}/quicktour.mdx (98%) diff --git a/docs/_toctree.yml b/docs/source/_toctree.yml similarity index 100% rename from docs/_toctree.yml rename to docs/source/_toctree.yml diff --git a/docs/index.mdx b/docs/source/index.mdx similarity index 75% rename from docs/index.mdx rename to docs/source/index.mdx index 4f5776f..008be12 100644 --- a/docs/index.mdx +++ b/docs/source/index.mdx @@ -24,21 +24,3 @@ Supported methods include: 2. Prefix Tuning: [Prefix-Tuning: Optimizing Continuous Prompts for Generation](https://aclanthology.org/2021.acl-long.353/), [P-Tuning v2: Prompt Tuning Can Be Comparable to Fine-tuning Universally Across Scales and Tasks](https://arxiv.org/pdf/2110.07602.pdf) 3. P-Tuning: [GPT Understands, Too](https://arxiv.org/pdf/2103.10385.pdf) 4. Prompt Tuning: [The Power of Scale for Parameter-Efficient Prompt Tuning](https://arxiv.org/pdf/2104.08691.pdf) - -## Getting started - -```python -from transformers import AutoModelForSeq2SeqLM -from peft import get_peft_config, get_peft_model, LoraConfig, TaskType - -model_name_or_path = "bigscience/mt0-large" -tokenizer_name_or_path = "bigscience/mt0-large" - -peft_config = LoraConfig(task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1) - -model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path) -model = get_peft_model(model, peft_config) -model.print_trainable_parameters() -# output: trainable params: 2359296 || all params: 1231940608 || trainable%: 0.19151053100118282 -``` - diff --git a/docs/install.mdx b/docs/source/install.mdx similarity index 100% rename from docs/install.mdx rename to docs/source/install.mdx diff --git a/docs/package_reference/config b/docs/source/package_reference/config similarity index 100% rename from docs/package_reference/config rename to docs/source/package_reference/config diff --git a/docs/package_reference/peft_model b/docs/source/package_reference/peft_model similarity index 100% rename from docs/package_reference/peft_model rename to docs/source/package_reference/peft_model diff --git a/docs/package_reference/tuners b/docs/source/package_reference/tuners similarity index 100% rename from docs/package_reference/tuners rename to docs/source/package_reference/tuners diff --git a/docs/quicktour.mdx b/docs/source/quicktour.mdx similarity index 98% rename from docs/quicktour.mdx rename to docs/source/quicktour.mdx index e0eb37f..a625aa8 100644 --- a/docs/quicktour.mdx +++ b/docs/source/quicktour.mdx @@ -29,9 +29,6 @@ from peft import LoraConfig, TaskType peft_config = LoraConfig(task_type=TaskType.SEQ_2_SEQ_LM, inference_mode=False, r=8, lora_alpha=32, lora_dropout=0.1) ``` -Here, `task_type` is the type of task you are training your model for. -For available task types, please refer [TaskType](package_reference/config#peft.config.TaskType). - 2. Load the base model you want to fine-tune. ```python From d8d1007732c464e4d86171f4741f8f2d1920d276 Mon Sep 17 00:00:00 2001 From: Vineet Kumar Date: Wed, 29 Mar 2023 18:50:14 +0530 Subject: [PATCH 5/7] Causal LM generation fix for prefix tuning: GPT2 model (#222) * expand attention mask after preparing generation inputs for prefix tuning * reformat * Update src/peft/peft_model.py Co-authored-by: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> * reformat as per black --------- Co-authored-by: Vineet Kumar Co-authored-by: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> --- src/peft/peft_model.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py index f73a66a..7491342 100644 --- a/src/peft/peft_model.py +++ b/src/peft/peft_model.py @@ -582,7 +582,13 @@ class PeftModelForCausalLM(PeftModel): else: if "input_ids" not in kwargs: raise ValueError("input_ids must be provided for Peft model generation") - if kwargs.get("attention_mask", None) is not None: + # For gpt2 models, we construct postion_ids on the fly by using attention mask, and position ids need to match input_shape. + # for prefix tuning, input shape is determined using `input_ids`. Thus we should not expand 'attention_mask' here + # for prompt tuning input_ids is not passed but a concatenated input_embeds is passed. Thus attention_mask needs to be of same size of num_virtual_tokens + input_ids + if kwargs.get("attention_mask", None) is not None and self.peft_config.peft_type in [ + PeftType.PROMPT_TUNING, + PeftType.P_TUNING, + ]: # concat prompt attention mask prefix_attention_mask = torch.ones( kwargs["input_ids"].shape[0], self.peft_config.num_virtual_tokens @@ -611,6 +617,14 @@ class PeftModelForCausalLM(PeftModel): def prepare_inputs_for_generation(self, *args, **kwargs): model_kwargs = self.base_model_prepare_inputs_for_generation(*args, **kwargs) if isinstance(self.peft_config, PromptLearningConfig): + if self.peft_config.peft_type == PeftType.PREFIX_TUNING: + prefix_attention_mask = torch.ones( + model_kwargs["input_ids"].shape[0], self.peft_config.num_virtual_tokens + ).to(model_kwargs["input_ids"].device) + model_kwargs["attention_mask"] = torch.cat( + (prefix_attention_mask, model_kwargs["attention_mask"]), dim=1 + ) + if model_kwargs["past_key_values"] is None and self.peft_config.peft_type == PeftType.PREFIX_TUNING: past_key_values = self.get_prompt(batch_size=model_kwargs["input_ids"].shape[0]) model_kwargs["past_key_values"] = past_key_values From df71b84341ae1ab3bc9b0d5f906d7a524850b63b Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Wed, 29 Mar 2023 15:28:38 +0200 Subject: [PATCH 6/7] [`CI`] Add more ci tests (#223) * add more tests * fix * add generate tests * make style * fix test * add -n * skip llama --- Makefile | 2 +- tests/test_peft_model.py | 39 ++++++++++++++++++++++++++++++++++----- tests/testing_common.py | 11 ++++++----- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 61549db..03ae1e0 100644 --- a/Makefile +++ b/Makefile @@ -17,4 +17,4 @@ style: doc-builder style src tests --max_len 119 test: - pytest tests/ \ No newline at end of file + pytest -n 3 tests/ \ No newline at end of file diff --git a/tests/test_peft_model.py b/tests/test_peft_model.py index 2ca4895..275a3cf 100644 --- a/tests/test_peft_model.py +++ b/tests/test_peft_model.py @@ -31,8 +31,14 @@ from .testing_common import PeftTestConfigManager # This has to be in the order: model_id, lora_kwargs, prefix_tuning_kwargs, prompt_encoder_kwargs, prompt_tuning_kwargs -PEFT_MODELS_TO_TEST = [ - ("hf-internal-testing/tiny-random-OPTForCausalLM", {"target_modules": ["q_proj", "v_proj"]}, {}, {}, {}), +PEFT_DECODER_MODELS_TO_TEST = [ + # ("HuggingFaceM4/tiny-random-LlamaForCausalLM", {}, {}, {}, {}), wait until the next `transformers` release + ("hf-internal-testing/tiny-random-OPTForCausalLM", {}, {}, {}, {}), + ("hf-internal-testing/tiny-random-GPTNeoXForCausalLM", {}, {}, {}, {}), + ("hf-internal-testing/tiny-random-GPT2LMHeadModel", {}, {}, {}, {}), + ("hf-internal-testing/tiny-random-BloomForCausalLM", {}, {}, {}, {}), + ("hf-internal-testing/tiny-random-gpt_neo", {}, {}, {}, {}), + ("hf-internal-testing/tiny-random-GPTJForCausalLM", {}, {}, {}, {}), ] @@ -48,7 +54,7 @@ class PeftModelTester(unittest.TestCase, PeftTestMixin): We use parametrized.expand for debugging purposes to test each model individually. """ - @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_MODELS_TO_TEST)) + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_DECODER_MODELS_TO_TEST)) def test_attributes_parametrized(self, test_name, model_id, config_cls, config_kwargs): self._test_model_attr(model_id, config_cls, config_kwargs) @@ -105,7 +111,7 @@ class PeftModelTester(unittest.TestCase, PeftTestMixin): self.assertTrue(dummy_output.requires_grad) - @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_MODELS_TO_TEST)) + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_DECODER_MODELS_TO_TEST)) def test_prepare_for_training_parametrized(self, test_name, model_id, config_cls, config_kwargs): self._test_prepare_for_training(model_id, config_cls, config_kwargs) @@ -151,6 +157,29 @@ class PeftModelTester(unittest.TestCase, PeftTestMixin): # check if `config.json` is not present self.assertFalse(os.path.exists(os.path.join(tmp_dirname, "config.json"))) - @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_MODELS_TO_TEST)) + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_DECODER_MODELS_TO_TEST)) def test_save_pretrained(self, test_name, model_id, config_cls, config_kwargs): self._test_save_pretrained(model_id, config_cls, config_kwargs) + + def _test_generate(self, model_id, config_cls, config_kwargs): + model = AutoModelForCausalLM.from_pretrained(model_id) + config = config_cls( + base_model_name_or_path=model_id, + **config_kwargs, + ) + model = get_peft_model(model, config) + model = model.to(self.torch_device) + + input_ids = torch.LongTensor([[1, 1, 1], [2, 1, 2]]).to(self.torch_device) + attention_mask = torch.LongTensor([[1, 1, 1], [1, 0, 1]]).to(self.torch_device) + + # check if `generate` works + _ = model.generate(input_ids=input_ids, attention_mask=attention_mask) + + with self.assertRaises(TypeError): + # check if `generate` raises an error if no positional arguments are passed + _ = model.generate(input_ids, attention_mask=attention_mask) + + @parameterized.expand(PeftTestConfigManager.get_grid_parameters(PEFT_DECODER_MODELS_TO_TEST)) + def test_generate(self, test_name, model_id, config_cls, config_kwargs): + self._test_generate(model_id, config_cls, config_kwargs) diff --git a/tests/testing_common.py b/tests/testing_common.py index dfdf1d8..96c0fdb 100644 --- a/tests/testing_common.py +++ b/tests/testing_common.py @@ -79,23 +79,24 @@ class ClassInstantier(OrderedDict): for model_tuple in model_list: model_id, lora_kwargs, prefix_tuning_kwargs, prompt_encoder_kwargs, prompt_tuning_kwargs = model_tuple for key, value in self.items(): + peft_method = value[1].copy() if key == "lora": # update value[1] if necessary if lora_kwargs is not None: - value[1].update(lora_kwargs) + peft_method.update(lora_kwargs) elif key == "prefix_tuning": # update value[1] if necessary if prefix_tuning_kwargs is not None: - value[1].update(prefix_tuning_kwargs) + peft_method.update(prefix_tuning_kwargs) elif key == "prompt_encoder": # update value[1] if necessary if prompt_encoder_kwargs is not None: - value[1].update(prompt_encoder_kwargs) + peft_method.update(prompt_encoder_kwargs) else: # update value[1] if necessary if prompt_tuning_kwargs is not None: - value[1].update(prompt_tuning_kwargs) - grid_parameters.append((f"test_{model_id}_{key}", model_id, value[0], value[1])) + peft_method.update(prompt_tuning_kwargs) + grid_parameters.append((f"test_{model_id}_{key}", model_id, value[0], peft_method)) return grid_parameters From d6c68ae1a5c17e3b4f1805233db3479d6033eaeb Mon Sep 17 00:00:00 2001 From: Aitor Gamarra <60578201+aitor-gamarra@users.noreply.github.com> Date: Wed, 29 Mar 2023 21:03:39 +0200 Subject: [PATCH 7/7] Show CONFIG_NAME instead of "config.json" --- src/peft/utils/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/peft/utils/config.py b/src/peft/utils/config.py index 3e2cf5b..2be3817 100644 --- a/src/peft/utils/config.py +++ b/src/peft/utils/config.py @@ -98,7 +98,7 @@ class PeftConfigMixin(PushToHubMixin): try: config_file = hf_hub_download(pretrained_model_name_or_path, CONFIG_NAME) except Exception: - raise ValueError(f"Can't find config.json at '{pretrained_model_name_or_path}'") + raise ValueError(f"Can't find '{CONFIG_NAME}' at '{pretrained_model_name_or_path}'") loaded_attributes = cls.from_json_file(config_file)