From 7ef47be5f5f8d608773312d2f3e037f073f27e3b Mon Sep 17 00:00:00 2001 From: tpoisonooo Date: Mon, 3 Apr 2023 14:02:13 +0800 Subject: [PATCH 1/5] Update other.py typo --- src/peft/utils/other.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/peft/utils/other.py b/src/peft/utils/other.py index 132b033..585da64 100644 --- a/src/peft/utils/other.py +++ b/src/peft/utils/other.py @@ -34,7 +34,7 @@ def prepare_model_for_int8_training( model, output_embedding_layer_name="lm_head", use_gradient_checkpointing=True, layer_norm_names=["layer_norm"] ): r""" - This method wrapps the entire protocol for preparing a model before running a training. This includes: + This method wraps the entire protocol for preparing a model before running a training. This includes: 1- Cast the layernorm in fp32 2- making output embedding layer require grads 3- Add the upcasting of the lm head to fp32 From 39cbd7d8ed6b2fc56442ada66fba32898cfd00aa Mon Sep 17 00:00:00 2001 From: Guspan Tanadi <36249910+guspan-tanadi@users.noreply.github.com> Date: Mon, 3 Apr 2023 16:13:33 +0700 Subject: [PATCH 2/5] docs: have fix bit typo README Improve readability --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5653c1a..0dc7c8a 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Hardware: Single A100 80GB GPU with CPU RAM above 64GB | bigscience/bloomz-7b1 (7B params) | OOM GPU | 32GB GPU / 3.8GB CPU | 18.1GB GPU / 35GB CPU | Performance of PEFT-LoRA tuned [`bigscience/T0_3B`](https://huggingface.co/bigscience/T0_3B) on [`ought/raft/twitter_complaints`](https://huggingface.co/datasets/ought/raft/viewer/twitter_complaints) leaderboard. -A point to note is that we didn't try to sequeeze performance by playing around with input instruction templates, LoRA hyperparams and other training related hyperparams. Also, we didn't use the larger 13B [mt0-xxl](https://huggingface.co/bigscience/mt0-xxl) model. +A point to note is that we didn't try to squeeze performance by playing around with input instruction templates, LoRA hyperparams and other training related hyperparams. Also, we didn't use the larger 13B [mt0-xxl](https://huggingface.co/bigscience/mt0-xxl) model. So, we are already seeing comparable performance to SoTA with parameter efficient tuning. Also, the final checkpoint size is just `19MB` in comparison to `11GB` size of the backbone [`bigscience/T0_3B`](https://huggingface.co/bigscience/T0_3B) model. | Submission Name | Accuracy | @@ -81,7 +81,7 @@ GPU memory required by different settings during training is given below. The fi Hardware: Single A100 80GB GPU with CPU RAM above 64GB -| Model | Full Finetuning | PEFT-LoRA | PEFT-LoRA with Gradient Checkpoitning | +| Model | Full Finetuning | PEFT-LoRA | PEFT-LoRA with Gradient Checkpointing | | --------- | ---- | ---- | ---- | | CompVis/stable-diffusion-v1-4 | 27.5GB GPU / 3.97GB CPU | 15.5GB GPU / 3.84GB CPU | 8.12GB GPU / 3.77GB CPU | @@ -148,7 +148,7 @@ Another example is fine-tuning [`roberta-large`](https://huggingface.co/roberta- ## PEFT + 🤗 Accelerate -PEFT models work with 🤗 Accelerate out of the box. Use 🤗 Accelerate for Distributed training on various hardware such as GPUs, Apple Silicon devices etc during training. +PEFT models work with 🤗 Accelerate out of the box. Use 🤗 Accelerate for Distributed training on various hardware such as GPUs, Apple Silicon devices, etc during training. Use 🤗 Accelerate for inferencing on consumer hardware with small resources. ### Example of PEFT model training using 🤗 Accelerate's DeepSpeed integration From 4ddb85ce1e2a25d11e5c32e485e2348df792da2c Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Mon, 3 Apr 2023 17:08:42 +0200 Subject: [PATCH 3/5] Update stale.py --- scripts/stale.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/stale.py b/scripts/stale.py index a0bd10a..e910135 100644 --- a/scripts/stale.py +++ b/scripts/stale.py @@ -28,6 +28,7 @@ LABELS_TO_EXEMPT = [ "feature request", "new model", "wip", + "PRs welcome to address this", ] @@ -59,4 +60,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() From 45d7aab39a0a580201209709cbc38d248cec0193 Mon Sep 17 00:00:00 2001 From: Stas Bekman Date: Mon, 3 Apr 2023 08:51:01 -0700 Subject: [PATCH 4/5] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5653c1a..dffb656 100644 --- a/README.md +++ b/README.md @@ -25,10 +25,10 @@ Seamlessly integrated with 🤗 Accelerate for large scale models leveraging Dee Supported methods: -1. LoRA: [LORA: LOW-RANK ADAPTATION OF LARGE LANGUAGE MODELS](https://arxiv.org/pdf/2106.09685.pdf) +1. LoRA: [LORA: LOW-RANK ADAPTATION OF LARGE LANGUAGE MODELS](https://arxiv.org/abs/2106.09685) 2. Prefix Tuning: [Prefix-Tuning: Optimizing Continuous Prompts for Generation](https://aclanthology.org/2021.acl-long.353/), [P-Tuning v2: Prompt Tuning Can Be Comparable to Fine-tuning Universally Across Scales and Tasks](https://arxiv.org/pdf/2110.07602.pdf) -3. P-Tuning: [GPT Understands, Too](https://arxiv.org/pdf/2103.10385.pdf) -4. Prompt Tuning: [The Power of Scale for Parameter-Efficient Prompt Tuning](https://arxiv.org/pdf/2104.08691.pdf) +3. P-Tuning: [GPT Understands, Too](https://arxiv.org/abs/2103.10385) +4. Prompt Tuning: [The Power of Scale for Parameter-Efficient Prompt Tuning](https://arxiv.org/abs/2104.08691) ## Getting started From ff9a1edbfd2d405b86d50a2e5299cc1bbd49d887 Mon Sep 17 00:00:00 2001 From: toncho11 Date: Mon, 3 Apr 2023 18:28:11 +0200 Subject: [PATCH 5/5] Fixing a bug where a wrong parameter name is used. --- src/peft/peft_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/peft/peft_model.py b/src/peft/peft_model.py index f9573bb..85757b7 100644 --- a/src/peft/peft_model.py +++ b/src/peft/peft_model.py @@ -164,7 +164,7 @@ class PeftModel(PushToHubMixin, torch.nn.Module): if getattr(model, "hf_device_map", None) is not None: device_map = kwargs.get("device_map", "auto") max_memory = kwargs.get("max_memory", None) - offload_dir = kwargs.get("offload_dir", None) + offload_dir = kwargs.get("offload_folder", None) offload_index = kwargs.get("offload_index", None) dispatch_model_kwargs = {}