From e4dcfaf1b356e399a7534a43afa2a78c1518b78a Mon Sep 17 00:00:00 2001 From: MKhalusova Date: Thu, 30 Mar 2023 11:30:55 -0400 Subject: [PATCH 1/6] task guide based on notebook --- docs/source/_toctree.yml | 4 + .../task_guides/image_classification_lora.mdx | 428 ++++++++++++++++++ 2 files changed, 432 insertions(+) create mode 100644 docs/source/task_guides/image_classification_lora.mdx diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index 211b83f..bb5a882 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -6,6 +6,10 @@ title: Quicktour - local: installation title: Installation +- title: Task Guides + sections: + - local: task_guides/image_classification_lora + title: Image classification using LoRA - title: Reference sections: - local: package_reference/peft_model diff --git a/docs/source/task_guides/image_classification_lora.mdx b/docs/source/task_guides/image_classification_lora.mdx new file mode 100644 index 0000000..17f00d3 --- /dev/null +++ b/docs/source/task_guides/image_classification_lora.mdx @@ -0,0 +1,428 @@ + + +# Fine-tuning for image classification using LoRA + +This guide demonstrates how to use LoRA, a low-rank approximation technique, to fine-tune an image classification model. +By using LoRA from 🤗 PEFT, we can reduce the number of trainable parameters in the model to only 0.77% of the original. + +LoRA achieves this reduction by adding low-rank "update matrices" to specific blocks of the model, such as the attention +blocks. During fine-tuning, only these matrices are trained, while the original model parameters are left unchanged. +At inference time, the update matrices are merged with the original model parameters to produce the final classification result. + +For more information on LoRA, please refer to the [original LoRA paper](https://arxiv.org/abs/2106.09685). + +## Install dependencies + +Install the libraries required for model training. To ensure you have access to all the latest features of 🤗 PEFT, +install it from source: + +```bash +pip install transformers accelerate evaluate datasets loralib git+https://github.com/huggingface/peft -q +``` + +Check the versions of all required libraries: + +```python +import transformers +import accelerate +import peft + +print(f"Transformers version: {transformers.__version__}") +print(f"Accelerate version: {accelerate.__version__}") +print(f"PEFT version: {peft.__version__}") +'Transformers version: 4.26.0' +'Accelerate version: 0.16.0' +'PEFT version: 0.1.0.dev0' +``` + +## Authenticate to share your model + +To share the fine-tuned model at the end of the training with the community, authenticate using your 🤗 token. +You can obtain your token from [here](https://huggingface.co/settings/token). + +```python +from huggingface_hub import notebook_login + +notebook_login() +``` + +## Select a model checkpoint to fine-tune + +Choose a model checkpoint from any of the model architectures supported for image classification. When in doubt, refer to +the [image classification task guide](https://huggingface.co/docs/transformers/v4.27.2/en/tasks/image_classification) in +🤗 Transformers documentation. + +```python +model_checkpoint = "google/vit-base-patch16-224-in21k" +``` + +## Load a dataset + +To keep this example's runtime short, let's only load the first 5000 instances from the training set of the Food-101 dataset: + +```python +from datasets import load_dataset + +dataset = load_dataset("food101", split="train[:5000]") +``` + +## Dataset Preparation + +To prepare the dataset for training and evaluation, create `label2id` and `id2label` dictionaries. These will come in +handy when performing inference and for metadata information: + +```python +labels = dataset.features["label"].names +label2id, id2label = dict(), dict() +for i, label in enumerate(labels): + label2id[label] = i + id2label[i] = label + +id2label[2] +'baklava' +``` + +Next, load the image processor of the model you're fine-tuning: + +```python +from transformers import AutoImageProcessor + +image_processor = AutoImageProcessor.from_pretrained(model_checkpoint) +``` + +The `image_processor` contains useful information on which size the training and evaluation images should be resized +to, as well as values that should be used to normalize the pixel values. Using the `image_processor`, prepare transformation +functions for the datasets. These functions will include data augmentation and pixel scaling: + +```python +from torchvision.transforms import ( + CenterCrop, + Compose, + Normalize, + RandomHorizontalFlip, + RandomResizedCrop, + Resize, + ToTensor, +) + +normalize = Normalize(mean=image_processor.image_mean, std=image_processor.image_std) +train_transforms = Compose( + [ + RandomResizedCrop(image_processor.size["height"]), + RandomHorizontalFlip(), + ToTensor(), + normalize, + ] +) + +val_transforms = Compose( + [ + Resize(image_processor.size["height"]), + CenterCrop(image_processor.size["height"]), + ToTensor(), + normalize, + ] +) + + +def preprocess_train(example_batch): + """Apply train_transforms across a batch.""" + example_batch["pixel_values"] = [train_transforms(image.convert("RGB")) for image in example_batch["image"]] + return example_batch + + +def preprocess_val(example_batch): + """Apply val_transforms across a batch.""" + example_batch["pixel_values"] = [val_transforms(image.convert("RGB")) for image in example_batch["image"]] + return example_batch +``` + +Split the dataset into training and validation sets: + +```python +splits = dataset.train_test_split(test_size=0.1) +train_ds = splits["train"] +val_ds = splits["test"] +``` + +Finally, set the transformation functions for the datasets accordingly: + +```python +train_ds.set_transform(preprocess_train) +val_ds.set_transform(preprocess_val) +``` + +## Load and prepare a model + +Before loading the model, let's define a helper function to check the total number of parameters a model has, as well +as how many of them are trainable. + +```python +def print_trainable_parameters(model): + trainable_params = 0 + all_param = 0 + for _, param in model.named_parameters(): + all_param += param.numel() + if param.requires_grad: + trainable_params += param.numel() + print( + f"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param:.2f}" + ) +``` + +It's important for to initialize the original model correctly as it will be used as a base to create a `PeftModel` you'll +actually fine-tune. Specify the `label2id` and `id2label` so that `AutoModelForImageClassification` can append a classification +head to the underlying model, adapted for this dataset. You should see the following output: + +``` +Some weights of ViTForImageClassification were not initialized from the model checkpoint at google/vit-base-patch16-224-in21k and are newly initialized: ['classifier.weight', 'classifier.bias'] +``` + +```python +from transformers import AutoModelForImageClassification, TrainingArguments, Trainer + +model = AutoModelForImageClassification.from_pretrained( + model_checkpoint, + label2id=label2id, + id2label=id2label, + ignore_mismatched_sizes=True, # provide this in case you're planning to fine-tune an already fine-tuned checkpoint +) +``` + +Before creating a `PeftModel`, you can check the number of trainable parameters in the original model: + +```python +print_trainable_parameters(model) +'trainable params: 85876325 || all params: 85876325 || trainable%: 100.00' +``` + +Next, use `PeftModel` to wrap the base model so that "update" matrices are added to the respective places. + +```python +from peft import LoraConfig, get_peft_model + +config = LoraConfig( + r=16, + lora_alpha=16, + target_modules=["query", "value"], + lora_dropout=0.1, + bias="none", + modules_to_save=["classifier"], +) +lora_model = get_peft_model(model, config) +print_trainable_parameters(lora_model) +'trainable params: 667493 || all params: 86466149 || trainable%: 0.77' +``` + +Let's unpack what's going on here. +To use LoRA, you need to specify the target modules to `LoraConfig` so that `get_peft_model()`` knows which modules +inside our model need to be amended with LoRA matrices. In this example, we're only interested in targeting the query and +value matrices of the attention blocks of the base model. Since the parameters corresponding to these matrices are "named" +with "query" and "value" respectively, we specify them accordingly in the `target_modules` argument of `LoraConfig`. + +We also specify `modules_to_save`. After wrapping the base model with `get_peft_model()` along with the `config`, we get +a new model where only the LoRA parameters are trainable (so-called "update matrices") while the pre-trained parameters +are kept frozen. However, we want the classifier parameters to be trained too when fine-tuning the base model on our +custom dataset. To ensure that the classifier parameters are also trained, we specify `modules_to_save`. This also +ensures that these modules are serialized alongside the LoRA trainable parameters when using utilities like `save_pretrained()` +and `push_to_hub()`. + +Here's what the other parameters mean: + +`r`: The dimension used by the LoRA update matrices. +`alpha`: Scaling factor. +`bias`: Specifies if the `bias` parameters should be trained. `None` denotes none of the `bias` parameters will be trained. + +`r` and `alpha` together control the total number of final trainable parameters when using LoRA, giving you the flexibility +to balance a trade-off between end performance and compute efficiency. + +By looking at the number of trainable parameters, you can see how many parameters we're actually training. Since the goal is +to achieve parameter-efficient fine-tuning, you should expect to see fewer trainable parameters in the `lora_model` +in comparison to the original model, which is indeed the case here. + +## Define training arguments + +For model fine-tuning, use [🤗 Trainer](https://huggingface.co/docs/transformers/main_classes/trainer). It accepts +several arguments which you can wrap using `TrainingArguments`. + +```python +from transformers import TrainingArguments, Trainer + + +model_name = model_checkpoint.split("/")[-1] +batch_size = 128 + +args = TrainingArguments( + f"{model_name}-finetuned-lora-food101", + remove_unused_columns=False, + evaluation_strategy="epoch", + save_strategy="epoch", + learning_rate=5e-3, + per_device_train_batch_size=batch_size, + gradient_accumulation_steps=4, + per_device_eval_batch_size=batch_size, + fp16=True, + num_train_epochs=5, + logging_steps=10, + load_best_model_at_end=True, + metric_for_best_model="accuracy", + push_to_hub=True, + label_names=["labels"], +) +``` + +Compared to fine-tuning the original model, you can use a larger batch size since there is only a handful of parameters to train. +You can also set a larger learning rate than the normal (1e-5 for example). + +This is a byproduct of the fact that the training affects only a small number of parameters. This can +potentially also reduce the need to conduct expensive hyperparameter tuning experiments. + +## Prepare evaluation metric + +```python +import numpy as np +import evaluate + +metric = evaluate.load("accuracy") + +# the compute_metrics function takes a Named Tuple as input: +# predictions, which are the logits of the model as Numpy arrays, +# and label_ids, which are the ground-truth labels as Numpy arrays. +def compute_metrics(eval_pred): + """Computes accuracy on a batch of predictions""" + predictions = np.argmax(eval_pred.predictions, axis=1) + return metric.compute(predictions=predictions, references=eval_pred.label_ids) + +``` + +## Define collation function + +A collation function is used by `Trainer` to gather a batch of training and evaluation examples and prepare them in a +format that is acceptable by the underlying model. + +```python +import torch + +def collate_fn(examples): + pixel_values = torch.stack([example["pixel_values"] for example in examples]) + labels = torch.tensor([example["label"] for example in examples]) + return {"pixel_values": pixel_values, "labels": labels} +``` + +## Train and evaluate + +Bring everything together - model, training arguments, data, collation function, etc. Then, start the training! + +```python +trainer = Trainer( + model, + args, + train_dataset=train_ds, + eval_dataset=val_ds, + tokenizer=image_processor, + compute_metrics=compute_metrics, + data_collator=collate_fn, +) +train_results = trainer.train() +``` + +In just a few minutes, the fine-tuned model shows 96% validation accuracy even on this small +subset of the training dataset. + +```python +trainer.evaluate(val_ds) +{'eval_loss': 0.14475855231285095, + 'eval_accuracy': 0.96, + 'eval_runtime': 3.5725, + 'eval_samples_per_second': 139.958, + 'eval_steps_per_second': 1.12, + 'epoch': 5.0} +``` + +## Share your model and run inference + +Once the fine-tuning is done, share the LoRA parameters with the community like so: + +```python +repo_name = f"sayakpaul/{model_name}-finetuned-lora-food101" +lora_model.push_to_hub(repo_name) +``` + +When calling `push_to_hub()` on the `lora_model`, only the LoRA parameters along with any modules specified in `modules_to_save` +are saved. Take a look at the [trained LoRA parameters](https://huggingface.co/sayakpaul/vit-base-patch16-224-in21k-finetuned-lora-food101/blob/main/adapter_model.bin). +You'll see that it's only 2.6 MB! This greatly helps with portability especially when using a very large model to fine-tune (such as [BLOOM](https://huggingface.co/bigscience/bloom). + +Next, let's see how to load the LoRA updated parameters along with our base model for inference. When you wrap a base model +with `PeftModel` that modifications are DONE in place. So to mitigate any concerns that might stem from in place modifications, +initialize the base model just like you did earlier and construct the inference model. + +```python +from peft import PeftConfig, PeftModel + + +config = PeftConfig.from_pretrained(repo_name) +model = model = AutoModelForImageClassification.from_pretrained( + config.base_model_name_or_path, + label2id=label2id, + id2label=id2label, + ignore_mismatched_sizes=True, # provide this in case you're planning to fine-tune an already fine-tuned checkpoint +) +# Load the LoRA model +inference_model = PeftModel.from_pretrained(model, repo_name) +``` + +Let's now fetch an example image for inference. + +```python +from PIL import Image +import requests + +url = "https://huggingface.co/datasets/sayakpaul/sample-datasets/resolve/main/beignets.jpeg" +image = Image.open(requests.get(url, stream=True).raw) +image +``` + +
+ image of beignets +
+ +First, instantiate an `image_processor` from the underlying model repo. + +```python +image_processor = AutoImageProcessor.from_pretrained(repo_name) +``` + +Then, prepare the example for inference. + +```python +encoding = image_processor(image.convert("RGB"), return_tensors="pt") +``` + +Finally, run inference! + +```python +with torch.no_grad(): + outputs = inference_model(**encoding) + logits = outputs.logits + +predicted_class_idx = logits.argmax(-1).item() +print("Predicted class:", inference_model.config.id2label[predicted_class_idx]) +'Predicted class: beignets' +``` + + + + + + + From d49cde41a7d62cc5e42b259430e95a6a1bc99e44 Mon Sep 17 00:00:00 2001 From: MKhalusova Date: Thu, 30 Mar 2023 12:09:28 -0400 Subject: [PATCH 2/6] make style --- .../task_guides/image_classification_lora.mdx | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/docs/source/task_guides/image_classification_lora.mdx b/docs/source/task_guides/image_classification_lora.mdx index 17f00d3..30cb3dc 100644 --- a/docs/source/task_guides/image_classification_lora.mdx +++ b/docs/source/task_guides/image_classification_lora.mdx @@ -40,9 +40,9 @@ import peft print(f"Transformers version: {transformers.__version__}") print(f"Accelerate version: {accelerate.__version__}") print(f"PEFT version: {peft.__version__}") -'Transformers version: 4.26.0' -'Accelerate version: 0.16.0' -'PEFT version: 0.1.0.dev0' +"Transformers version: 4.26.0" +"Accelerate version: 0.16.0" +"PEFT version: 0.1.0.dev0" ``` ## Authenticate to share your model @@ -89,7 +89,7 @@ for i, label in enumerate(labels): id2label[i] = label id2label[2] -'baklava' +"baklava" ``` Next, load the image processor of the model you're fine-tuning: @@ -203,7 +203,7 @@ Before creating a `PeftModel`, you can check the number of trainable parameters ```python print_trainable_parameters(model) -'trainable params: 85876325 || all params: 85876325 || trainable%: 100.00' +"trainable params: 85876325 || all params: 85876325 || trainable%: 100.00" ``` Next, use `PeftModel` to wrap the base model so that "update" matrices are added to the respective places. @@ -221,7 +221,7 @@ config = LoraConfig( ) lora_model = get_peft_model(model, config) print_trainable_parameters(lora_model) -'trainable params: 667493 || all params: 86466149 || trainable%: 0.77' +"trainable params: 667493 || all params: 86466149 || trainable%: 0.77" ``` Let's unpack what's going on here. @@ -295,6 +295,7 @@ import evaluate metric = evaluate.load("accuracy") + # the compute_metrics function takes a Named Tuple as input: # predictions, which are the logits of the model as Numpy arrays, # and label_ids, which are the ground-truth labels as Numpy arrays. @@ -302,7 +303,6 @@ def compute_metrics(eval_pred): """Computes accuracy on a batch of predictions""" predictions = np.argmax(eval_pred.predictions, axis=1) return metric.compute(predictions=predictions, references=eval_pred.label_ids) - ``` ## Define collation function @@ -313,6 +313,7 @@ format that is acceptable by the underlying model. ```python import torch + def collate_fn(examples): pixel_values = torch.stack([example["pixel_values"] for example in examples]) labels = torch.tensor([example["label"] for example in examples]) @@ -341,12 +342,14 @@ subset of the training dataset. ```python trainer.evaluate(val_ds) -{'eval_loss': 0.14475855231285095, - 'eval_accuracy': 0.96, - 'eval_runtime': 3.5725, - 'eval_samples_per_second': 139.958, - 'eval_steps_per_second': 1.12, - 'epoch': 5.0} +{ + "eval_loss": 0.14475855231285095, + "eval_accuracy": 0.96, + "eval_runtime": 3.5725, + "eval_samples_per_second": 139.958, + "eval_steps_per_second": 1.12, + "epoch": 5.0, +} ``` ## Share your model and run inference @@ -417,7 +420,7 @@ with torch.no_grad(): predicted_class_idx = logits.argmax(-1).item() print("Predicted class:", inference_model.config.id2label[predicted_class_idx]) -'Predicted class: beignets' +"Predicted class: beignets" ``` From 9ced552e65af707d8524a8e8f622c9f5475d09c4 Mon Sep 17 00:00:00 2001 From: MKhalusova Date: Thu, 30 Mar 2023 12:29:29 -0400 Subject: [PATCH 3/6] doc building fixes --- docs/source/_toctree.yml | 40 ++++++++++---------- docs/source/package_reference/config | 0 docs/source/package_reference/config.mdx | 1 + docs/source/package_reference/peft_model | 0 docs/source/package_reference/peft_model.mdx | 1 + docs/source/package_reference/tuners | 0 docs/source/package_reference/tuners.mdx | 1 + 7 files changed, 24 insertions(+), 19 deletions(-) delete mode 100644 docs/source/package_reference/config create mode 100644 docs/source/package_reference/config.mdx delete mode 100644 docs/source/package_reference/peft_model create mode 100644 docs/source/package_reference/peft_model.mdx delete mode 100644 docs/source/package_reference/tuners create mode 100644 docs/source/package_reference/tuners.mdx diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index bb5a882..1255844 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -1,20 +1,22 @@ -- title: Get Started - sections: - - local: index - title: 🤗 PEFT - - local: quicktour - title: Quicktour - - local: installation - title: Installation -- title: Task Guides - sections: - - local: task_guides/image_classification_lora - title: Image classification using LoRA +- title: Get started + sections: + - local: index + title: 🤗 PEFT + - local: quicktour + title: Quicktour + - local: install + title: Installation + +- title: Task guides + sections: + - local: task_guides/image_classification_lora + title: Image classification using LoRA + - title: Reference - sections: - - local: package_reference/peft_model - title: PEFT model - - local: package_reference/configs - title: Configuration - - local: package_reference/tuners - title: Tuners \ No newline at end of file + sections: + - local: package_reference/peft_model + title: PEFT model + - local: package_reference/config + title: Configuration + - local: package_reference/tuners + title: Tuners \ No newline at end of file diff --git a/docs/source/package_reference/config b/docs/source/package_reference/config deleted file mode 100644 index e69de29..0000000 diff --git a/docs/source/package_reference/config.mdx b/docs/source/package_reference/config.mdx new file mode 100644 index 0000000..af4abbf --- /dev/null +++ b/docs/source/package_reference/config.mdx @@ -0,0 +1 @@ +# Configuration \ No newline at end of file diff --git a/docs/source/package_reference/peft_model b/docs/source/package_reference/peft_model deleted file mode 100644 index e69de29..0000000 diff --git a/docs/source/package_reference/peft_model.mdx b/docs/source/package_reference/peft_model.mdx new file mode 100644 index 0000000..f6789d4 --- /dev/null +++ b/docs/source/package_reference/peft_model.mdx @@ -0,0 +1 @@ +# PEFT model \ No newline at end of file diff --git a/docs/source/package_reference/tuners b/docs/source/package_reference/tuners deleted file mode 100644 index e69de29..0000000 diff --git a/docs/source/package_reference/tuners.mdx b/docs/source/package_reference/tuners.mdx new file mode 100644 index 0000000..1e60f16 --- /dev/null +++ b/docs/source/package_reference/tuners.mdx @@ -0,0 +1 @@ +# Tuners \ No newline at end of file From 8a6004232ba0ef9658285aa5a035ababd489a9db Mon Sep 17 00:00:00 2001 From: Maria Khalusova Date: Fri, 31 Mar 2023 08:55:31 -0400 Subject: [PATCH 4/6] Apply suggestions from code review Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- .../task_guides/image_classification_lora.mdx | 49 +++++++++---------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/docs/source/task_guides/image_classification_lora.mdx b/docs/source/task_guides/image_classification_lora.mdx index 30cb3dc..4fca064 100644 --- a/docs/source/task_guides/image_classification_lora.mdx +++ b/docs/source/task_guides/image_classification_lora.mdx @@ -10,7 +10,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express o specific language governing permissions and limitations under the License. --> -# Fine-tuning for image classification using LoRA +# Image classification using LoRA This guide demonstrates how to use LoRA, a low-rank approximation technique, to fine-tune an image classification model. By using LoRA from 🤗 PEFT, we can reduce the number of trainable parameters in the model to only 0.77% of the original. @@ -27,10 +27,10 @@ Install the libraries required for model training. To ensure you have access to install it from source: ```bash -pip install transformers accelerate evaluate datasets loralib git+https://github.com/huggingface/peft -q +!pip install transformers accelerate evaluate datasets loralib git+https://github.com/huggingface/peft -q ``` -Check the versions of all required libraries: +Check the versions of all required libraries to make sure you are up to date: ```python import transformers @@ -48,7 +48,7 @@ print(f"PEFT version: {peft.__version__}") ## Authenticate to share your model To share the fine-tuned model at the end of the training with the community, authenticate using your 🤗 token. -You can obtain your token from [here](https://huggingface.co/settings/token). +You can obtain your token from your [account settings](https://huggingface.co/settings/token). ```python from huggingface_hub import notebook_login @@ -58,7 +58,7 @@ notebook_login() ## Select a model checkpoint to fine-tune -Choose a model checkpoint from any of the model architectures supported for image classification. When in doubt, refer to +Choose a model checkpoint from any of the model architectures supported for [image classification](https://huggingface.co/models?pipeline_tag=image-classification&sort=downloads). When in doubt, refer to the [image classification task guide](https://huggingface.co/docs/transformers/v4.27.2/en/tasks/image_classification) in 🤗 Transformers documentation. @@ -68,7 +68,7 @@ model_checkpoint = "google/vit-base-patch16-224-in21k" ## Load a dataset -To keep this example's runtime short, let's only load the first 5000 instances from the training set of the Food-101 dataset: +To keep this example's runtime short, let's only load the first 5000 instances from the training set of the [Food-101 dataset](https://huggingface.co/datasets/food101): ```python from datasets import load_dataset @@ -76,7 +76,7 @@ from datasets import load_dataset dataset = load_dataset("food101", split="train[:5000]") ``` -## Dataset Preparation +## Dataset preparation To prepare the dataset for training and evaluation, create `label2id` and `id2label` dictionaries. These will come in handy when performing inference and for metadata information: @@ -180,8 +180,8 @@ def print_trainable_parameters(model): ) ``` -It's important for to initialize the original model correctly as it will be used as a base to create a `PeftModel` you'll -actually fine-tune. Specify the `label2id` and `id2label` so that `AutoModelForImageClassification` can append a classification +It's important to initialize the original model correctly as it will be used as a base to create the `PeftModel` you'll +actually fine-tune. Specify the `label2id` and `id2label` so that [`~transformers.AutoModelForImageClassification`] can append a classification head to the underlying model, adapted for this dataset. You should see the following output: ``` @@ -206,7 +206,7 @@ print_trainable_parameters(model) "trainable params: 85876325 || all params: 85876325 || trainable%: 100.00" ``` -Next, use `PeftModel` to wrap the base model so that "update" matrices are added to the respective places. +Next, use `get_peft_model` to wrap the base model so that "update" matrices are added to the respective places. ```python from peft import LoraConfig, get_peft_model @@ -225,10 +225,10 @@ print_trainable_parameters(lora_model) ``` Let's unpack what's going on here. -To use LoRA, you need to specify the target modules to `LoraConfig` so that `get_peft_model()`` knows which modules +To use LoRA, you need to specify the target modules in `LoraConfig` so that `get_peft_model()` knows which modules inside our model need to be amended with LoRA matrices. In this example, we're only interested in targeting the query and value matrices of the attention blocks of the base model. Since the parameters corresponding to these matrices are "named" -with "query" and "value" respectively, we specify them accordingly in the `target_modules` argument of `LoraConfig`. +"query" and "value" respectively, we specify them accordingly in the `target_modules` argument of `LoraConfig`. We also specify `modules_to_save`. After wrapping the base model with `get_peft_model()` along with the `config`, we get a new model where only the LoRA parameters are trainable (so-called "update matrices") while the pre-trained parameters @@ -239,9 +239,9 @@ and `push_to_hub()`. Here's what the other parameters mean: -`r`: The dimension used by the LoRA update matrices. -`alpha`: Scaling factor. -`bias`: Specifies if the `bias` parameters should be trained. `None` denotes none of the `bias` parameters will be trained. +- `r`: The dimension used by the LoRA update matrices. +- `alpha`: Scaling factor. +- `bias`: Specifies if the `bias` parameters should be trained. `None` denotes none of the `bias` parameters will be trained. `r` and `alpha` together control the total number of final trainable parameters when using LoRA, giving you the flexibility to balance a trade-off between end performance and compute efficiency. @@ -252,8 +252,8 @@ in comparison to the original model, which is indeed the case here. ## Define training arguments -For model fine-tuning, use [🤗 Trainer](https://huggingface.co/docs/transformers/main_classes/trainer). It accepts -several arguments which you can wrap using `TrainingArguments`. +For model fine-tuning, use [`~transformers.Trainer`]. It accepts +several arguments which you can wrap using [`~transformers.TrainingArguments`]. ```python from transformers import TrainingArguments, Trainer @@ -281,11 +281,10 @@ args = TrainingArguments( ) ``` -Compared to fine-tuning the original model, you can use a larger batch size since there is only a handful of parameters to train. +Compared to non-PEFT methods, you can use a larger batch size since there are fewer parameters to train. You can also set a larger learning rate than the normal (1e-5 for example). -This is a byproduct of the fact that the training affects only a small number of parameters. This can -potentially also reduce the need to conduct expensive hyperparameter tuning experiments. +This can potentially also reduce the need to conduct expensive hyperparameter tuning experiments. ## Prepare evaluation metric @@ -307,7 +306,7 @@ def compute_metrics(eval_pred): ## Define collation function -A collation function is used by `Trainer` to gather a batch of training and evaluation examples and prepare them in a +A collation function is used by [`~transformers.Trainer`] to gather a batch of training and evaluation examples and prepare them in a format that is acceptable by the underlying model. ```python @@ -361,12 +360,12 @@ repo_name = f"sayakpaul/{model_name}-finetuned-lora-food101" lora_model.push_to_hub(repo_name) ``` -When calling `push_to_hub()` on the `lora_model`, only the LoRA parameters along with any modules specified in `modules_to_save` +When calling [`~transformers.PreTrainedModel.push_to_hub`] on the `lora_model`, only the LoRA parameters along with any modules specified in `modules_to_save` are saved. Take a look at the [trained LoRA parameters](https://huggingface.co/sayakpaul/vit-base-patch16-224-in21k-finetuned-lora-food101/blob/main/adapter_model.bin). -You'll see that it's only 2.6 MB! This greatly helps with portability especially when using a very large model to fine-tune (such as [BLOOM](https://huggingface.co/bigscience/bloom). +You'll see that it's only 2.6 MB! This greatly helps with portability, especially when using a very large model to fine-tune (such as [BLOOM](https://huggingface.co/bigscience/bloom)). Next, let's see how to load the LoRA updated parameters along with our base model for inference. When you wrap a base model -with `PeftModel` that modifications are DONE in place. So to mitigate any concerns that might stem from in place modifications, +with `PeftModel`, modifications are done *in-place*. To mitigate any concerns that might stem from in-place modifications, initialize the base model just like you did earlier and construct the inference model. ```python @@ -374,7 +373,7 @@ from peft import PeftConfig, PeftModel config = PeftConfig.from_pretrained(repo_name) -model = model = AutoModelForImageClassification.from_pretrained( +model = AutoModelForImageClassification.from_pretrained( config.base_model_name_or_path, label2id=label2id, id2label=id2label, From de2a46a2f9abbd653b350914dca1395c134f8943 Mon Sep 17 00:00:00 2001 From: MKhalusova Date: Fri, 31 Mar 2023 09:29:41 -0400 Subject: [PATCH 5/6] version fix --- docs/source/task_guides/image_classification_lora.mdx | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/source/task_guides/image_classification_lora.mdx b/docs/source/task_guides/image_classification_lora.mdx index 4fca064..8c82ea6 100644 --- a/docs/source/task_guides/image_classification_lora.mdx +++ b/docs/source/task_guides/image_classification_lora.mdx @@ -23,11 +23,10 @@ For more information on LoRA, please refer to the [original LoRA paper](https:// ## Install dependencies -Install the libraries required for model training. To ensure you have access to all the latest features of 🤗 PEFT, -install it from source: +Install the libraries required for model training: ```bash -!pip install transformers accelerate evaluate datasets loralib git+https://github.com/huggingface/peft -q +!pip install transformers accelerate evaluate datasets loralib peft -q ``` Check the versions of all required libraries to make sure you are up to date: @@ -40,9 +39,9 @@ import peft print(f"Transformers version: {transformers.__version__}") print(f"Accelerate version: {accelerate.__version__}") print(f"PEFT version: {peft.__version__}") -"Transformers version: 4.26.0" -"Accelerate version: 0.16.0" -"PEFT version: 0.1.0.dev0" +"Transformers version: 4.27.4" +"Accelerate version: 0.18.0" +"PEFT version: 0.2.0" ``` ## Authenticate to share your model From 221b39256db469607aa9557bab673414f17f1a55 Mon Sep 17 00:00:00 2001 From: MKhalusova Date: Fri, 31 Mar 2023 09:34:00 -0400 Subject: [PATCH 6/6] feedback addressed --- docs/source/task_guides/image_classification_lora.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/task_guides/image_classification_lora.mdx b/docs/source/task_guides/image_classification_lora.mdx index 8c82ea6..cfbbeca 100644 --- a/docs/source/task_guides/image_classification_lora.mdx +++ b/docs/source/task_guides/image_classification_lora.mdx @@ -294,15 +294,15 @@ import evaluate metric = evaluate.load("accuracy") -# the compute_metrics function takes a Named Tuple as input: -# predictions, which are the logits of the model as Numpy arrays, -# and label_ids, which are the ground-truth labels as Numpy arrays. def compute_metrics(eval_pred): """Computes accuracy on a batch of predictions""" predictions = np.argmax(eval_pred.predictions, axis=1) return metric.compute(predictions=predictions, references=eval_pred.label_ids) ``` +The `compute_metrics` function takes a named tuple as input: `predictions`, which are the logits of the model as Numpy arrays, +and `label_ids`, which are the ground-truth labels as Numpy arrays. + ## Define collation function A collation function is used by [`~transformers.Trainer`] to gather a batch of training and evaluation examples and prepare them in a