✨

README.md

@@ -59,7 +59,20 @@ So, we are already seeing comparable performance to SoTA with parameter effcient
 
 **Therefore, we can see that performance comparable to SoTA is achievable by PET methods with consumer hardware such as 16GB and 24GB GPUs.**
 
-### Parameter Efficient Tuning of Diffusion Models [ToDo]
+### Parameter Efficient Tuning of Diffusion Models
+
+GPU memory and training times required by different settings during training are given below. The final checkpoint size being `8.8 MB`.
+
+Hardware: Single A100 80GB GPU with CPU RAM above 64G
+
+|   Model         | Full Finetuning | PET-LoRA  |
+| --------- | ---- | ---- | ---- |
+| CompVis/stable-diffusion-v1-4 | 66.4GB GPU / 3.97GB CPU / 22 minutes  | 15.5GB GPU / 3.84GB CPU / 5.5 minutes | 
+
+
+**Training**
+An example of using LoRA for parameter efficient dreambooth training is given in `~examples/lora_dreambooth/train_dreambooth.py`
+
 ```bash
 export MODEL_NAME="stabilityai/stable-diffusion-2-1" #"CompVis/stable-diffusion-v1-4"
 export INSTANCE_DIR="path-to-instance-images"

examples/lora_dreambooth/train_dreambooth.py

@@ -10,32 +10,30 @@ import threading
 import warnings
 from pathlib import Path
 from typing import Optional
-import psutil
 
 import torch
 import torch.nn.functional as F
 import torch.utils.checkpoint
-from torch.utils.data import Dataset
-from dataclasses import asdict
-
-import datasets
-import diffusers
 import transformers
 from accelerate import Accelerator
 from accelerate.logging import get_logger
 from accelerate.utils import set_seed
+from torch.utils.data import Dataset
+from transformers import AutoTokenizer, PretrainedConfig
+
+import datasets
+import diffusers
+import psutil
 from diffusers import AutoencoderKL, DDPMScheduler, DiffusionPipeline, UNet2DConditionModel
-from diffusers import DDPMScheduler, PNDMScheduler, StableDiffusionPipeline
-from diffusers.pipelines.stable_diffusion import StableDiffusionSafetyChecker
 from diffusers.optimization import get_scheduler
 from diffusers.utils import check_min_version
 from diffusers.utils.import_utils import is_xformers_available
 from huggingface_hub import HfFolder, Repository, whoami
+from pet import LoRAConfig, LoRAModel, get_pet_model_state_dict
 from PIL import Image
 from torchvision import transforms
 from tqdm.auto import tqdm
-from transformers import AutoTokenizer, PretrainedConfig, CLIPFeatureExtractor
-from pet import LoRAModel, LoRAConfig, get_pet_model_state_dict
+
 
 # Will error if the minimal version of diffusers is not installed. Remove at your own risks.
 check_min_version("0.10.0.dev0")
@@ -632,7 +630,7 @@ def main(args):
                 repo_name = get_full_repo_name(Path(args.output_dir).name, token=args.hub_token)
             else:
                 repo_name = args.hub_model_id
-            repo = Repository(args.output_dir, clone_from=repo_name)
+            repo = Repository(args.output_dir, clone_from=repo_name)  # noqa: F841
 
             with open(os.path.join(args.output_dir, ".gitignore"), "w+") as gitignore:
                 if "step_*" not in gitignore:
@@ -670,7 +668,6 @@ def main(args):
     unet = UNet2DConditionModel.from_pretrained(
         args.pretrained_model_name_or_path, subfolder="unet", revision=args.revision
     )
-    feature_extractor = CLIPFeatureExtractor.from_pretrained("openai/clip-vit-base-patch32")
 
     if args.use_lora:
         config = LoRAConfig(

src/pet/tuners/lora.py

@@ -1,7 +1,7 @@
-from enum import Enum
 import math
 import warnings
-from dataclasses import dataclass, field, asdict
+from dataclasses import asdict, dataclass, field
+from enum import Enum
 from typing import List, Optional
 
 import torch