mirror of
https://github.com/wassname/peft.git
synced 2026-10-29 13:11:48 +08:00
add code and update README
This commit is contained in:
Sourab Mangrulkar committed
1 parent
49dc9d83c4
commit
71d6c074de
4 files changed
+1443
-13
No files matched your search
@@ -60,6 +60,35 @@ So, we are already seeing comparable performance to SoTA with parameter effcient
|
||||
**Therefore, we can see that performance comparable to SoTA is achievable by PET methods with consumer hardware such as 16GB and 24GB GPUs.**
|
||||
|
||||
### Parameter Efficient Tuning of Diffusion Models [ToDo]
|
||||
```bash
|
||||
export MODEL_NAME="stabilityai/stable-diffusion-2-1" #"CompVis/stable-diffusion-v1-4"
|
||||
export INSTANCE_DIR="path-to-instance-images"
|
||||
export CLASS_DIR="path-to-class-images"
|
||||
export OUTPUT_DIR="path-to-save-model"
|
||||
|
||||
accelerate launch train_dreambooth.py \
|
||||
--pretrained_model_name_or_path=$MODEL_NAME \
|
||||
--instance_data_dir=$INSTANCE_DIR \
|
||||
--class_data_dir=$CLASS_DIR \
|
||||
--output_dir=$OUTPUT_DIR \
|
||||
--train_text_encoder \
|
||||
--with_prior_preservation --prior_loss_weight=1.0 \
|
||||
--instance_prompt="a photo of sks dog" \
|
||||
--class_prompt="a photo of dog" \
|
||||
--resolution=512 \
|
||||
--train_batch_size=1 \
|
||||
--lr_scheduler="constant" \
|
||||
--lr_warmup_steps=0 \
|
||||
--num_class_images=200 \
|
||||
--use_lora \
|
||||
--lora_r 16 \
|
||||
--lora_alpha 27 \
|
||||
--lora_text_encoder_r 16 \
|
||||
--lora_text_encoder_alpha 17 \
|
||||
--learning_rate=1e-4 \
|
||||
--gradient_accumulation_steps=1 \
|
||||
--max_train_steps=800
|
||||
```
|
||||
|
||||
### Parameter Efficient Tuning of LLMs for RLHF components such as Ranker and Policy [ToDo]
|
||||
|
||||
|
||||
@@ -0,0 +1,398 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"id": "acab479f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import argparse\n",
|
||||
"import gc\n",
|
||||
"import hashlib\n",
|
||||
"import itertools\n",
|
||||
"import logging\n",
|
||||
"import math\n",
|
||||
"import os\n",
|
||||
"import threading\n",
|
||||
"import warnings\n",
|
||||
"from pathlib import Path\n",
|
||||
"from typing import Optional\n",
|
||||
"import psutil\n",
|
||||
"import json\n",
|
||||
"\n",
|
||||
"import torch\n",
|
||||
"import torch.nn.functional as F\n",
|
||||
"import torch.utils.checkpoint\n",
|
||||
"from torch.utils.data import Dataset\n",
|
||||
"\n",
|
||||
"import datasets\n",
|
||||
"import diffusers\n",
|
||||
"import transformers\n",
|
||||
"from accelerate import Accelerator\n",
|
||||
"from accelerate.logging import get_logger\n",
|
||||
"from accelerate.utils import set_seed\n",
|
||||
"from diffusers import AutoencoderKL, DDPMScheduler, DiffusionPipeline, UNet2DConditionModel\n",
|
||||
"from diffusers import DDPMScheduler, PNDMScheduler, StableDiffusionPipeline\n",
|
||||
"from diffusers.pipelines.stable_diffusion import StableDiffusionSafetyChecker\n",
|
||||
"from diffusers.optimization import get_scheduler\n",
|
||||
"from diffusers.utils import check_min_version\n",
|
||||
"from diffusers.utils.import_utils import is_xformers_available\n",
|
||||
"from huggingface_hub import HfFolder, Repository, whoami\n",
|
||||
"from PIL import Image\n",
|
||||
"from torchvision import transforms\n",
|
||||
"from tqdm.auto import tqdm\n",
|
||||
"from transformers import AutoTokenizer, PretrainedConfig, CLIPFeatureExtractor\n",
|
||||
"from pet import LoRAModel, LoRAConfig, get_pet_model_state_dict, set_pet_model_state_dict\n",
|
||||
"\n",
|
||||
"# Will error if the minimal version of diffusers is not installed. Remove at your own risks.\n",
|
||||
"check_min_version(\"0.10.0.dev0\")\n",
|
||||
"\n",
|
||||
"logger = get_logger(__name__)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"MODEL_NAME=\"CompVis/stable-diffusion-v1-4\"#\"stabilityai/stable-diffusion-2-1-base\"\n",
|
||||
"INSTANCE_PROMPT = \"a photo of sks dog\"\n",
|
||||
"ckpt_dir = \"/home/sourab/temp/sd_dog_dreambooth/\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "328161b7",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "5eafd408382e433fa904adcaad256551",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Fetching 16 files: 0%| | 0/16 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/home/sourab/transformers/src/transformers/models/clip/feature_extraction_clip.py:28: FutureWarning: The class CLIPFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please use CLIPImageProcessor instead.\n",
|
||||
" warnings.warn(\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "a15b91a5745548f9b52c67c38e9307b4",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/50 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAIAAAB7GkOtAAEAAElEQVR4nCz9x9Ou2Xoe9q0cnvzGL+zYu9OJAA+SABJgoUSbksUyq1x2+T/0wK7SQFLRskwKAAEIBhHPOZ279+7e6UtvfPLKa3lwNL5Ha3qt33Xf8I/+5Y+nuStERiVSk04W/PgnP+6Vebi/D94lmADlwce576d2YJkkhDpjI4AgpSfPrz/56W/92f/yv2AYnfHdOCEA66bYXq84IZBTzMXc90WeP338ZLW+yIsyZ3LoRxf0IhP7Y3van8qmyevFcrEkjFsflQu3b77p++PQtZSJrFo5M92/f3veHwAmEEAhWVGWMcbkgXeuKstFU/2rP/3jd3f3//C3f//k+rpaVCGGrz//+nhot5tVhHEaRsZRLmkmuJDy8fMnkItTO37/9StvXQSxb/eSsn60XWtlU24uN6t1ebh7MNYIQpoqu3hyeTiqvuujBwAgPY+n8+CCzySjHBMMAEogmqZeL9fL1cXF3f0xAZgAbY/n/c3Dv/lv/7W14YfX7402CMe6qZaL6umL69v73Q8vXzHuKUEgAsKy9jirYdw+ugwBnfZ7Jgih/LjfF1VBMBSS7ffHR4+uFosmgXTedxCg71++sgYgSLt55lRWVbZYZNaMxoa26yEClGLKoGjkxQeP81LeHw7W+XlWCcVoHRNMcCoJrBfFOKih00oFZwPCMDlfFvLiYqVU3L3bWQtQghxFAJE2gTK2Xi3zPE8I7e7ujPbROSpYColRTGg2j5OzerFc5WV22O2myWKaijKHEJRlTigRmQw+3bx7DyDJi5wzIoUwSj883GRZlWJkEudFNhvz7u3tONm6rB5fbopMxBSnfvaz4ZwhRHcP57EdPvn4iRT4/e0hJEgIXy0bH+I4zwWnQc+LbHG5uVyvFjLLgldJW0iKiODhPOz7dvnoyfevv/6bv/27iPBiUyfko/eJYURgSpAVWQpktSqsd33bjcMQAc6KCgLgrAMJUJJgAsZaZz0XIiUn64JzSgmBlHuEJj0b66KJBEIEkbdusa5hBKofpmGsi6xZrKwJpeS7210/jQHEmIAxpsjl0w+vnzz/xKrhl3//96e7U8CE8ULQrKmyxaKZlDZW14tmHofd/YM1hlAOQSIMZQVnAtfrZSZY8EQPBmD6/edvl9vV4XC4e78rKoIRjSEghAljIAUCoXEhRYc5IYxDDojkLINUEMpoik5ySTnBTDIhUvLGRYwwhsAMkzNOFGW9rFCCczd17Uw5K6tCTePtq+8QwsOgIUiYUMoIgohzlgBYNjVjvKgrQrgy9u7tje7maZqSD1TQZx881+NojYWQYEG1VYyzcVIUk6quypzClLyzWvV1zZ88e6oBOBzPw7mrinKJOfXx6WL9Oz//9OHmpq7r5CNnHGNICTXG3r+/f/fQ3Q/jm/NwPwyjcbKpfCKXm+rJ1fbZ4ydPnl7jojzujofbNx+/uCYo++HlZ6/f/WATikze3O7m/rx5tGCLatLOJ5QIoMjhALxNKSQA0Pvb4wcfPP2d3//52PfzMAQAinqRLRaHc/vZ3/6tmVVTVXmRcyGNjXe3tySGy+Xy6aPt5cWmHfrvv3/dns+U0ACRD+R8GiBGGERRSj3r6CwiVGmjJo0Z22y3KAIXTEgWOn+xbFZ1ngB7/eZ4d+yItXUhIsIBYc4EEYxB9PGzazKOQ3ceD3CuSkIp0pP7u3/4Z8L4MA5ZJkFEsxkI5wyxFx99SBmHiBjnhrFXsx569dk//nJR5iE4nJN6WQghCaFZwTnj28cX7ThbOzSrCknqQTLan48HwiClbAwoERy58EjyvAlM3uz23aFVWh3v3zqnvFWY+WbxFEN0sdxG7Q9de/fuYbGpB2VzkW3XG+Pcw/1RDfrzf/5iMHN3blfrbRlQ0Ma5qLS+vX1w3k99Rylcb5YXjy4yyo1BIEQK2aOrx3UuheCfff7P9/d7hOhyJRJA7c3u9PbWGjWM04sXTw11x117dxzmYb7YXgSfgotZWUxqjBAKIU6Hk9bzelMhzk/HYZydDUBmxS9+++dXV8+P929//cWv3r69GUYVE8DQJeidH+/u3zHBIILe+boqKRXGpHJRnfv266++/dFPfvKjn/xUq9E5h0Ac1Pz23W1K4emjR+1pTB5cXV8tKkIJnTb6/v5gQizyerlcUApKIWGePRyPZVNGEA93u3yR+35+Iqj1sMjKSEE6Q0ATgskFa5yZJ30eOoSJ1t665K2llMEErEfTHISUlFOQgpAlR4hCYFwwxintJMcQweDCNA4QwKGfszwTXFJCsyw/anPuh0FrBFlWCaMmozwioO2GIi8wFlZbmRfaWspoWdZlXrrceA/6cXDKOotjwhClZlkzrqIHbds7bcdeSc6ur66qokwJ5EUzDWPf98eTO7VjN89lXk6zYQTPw/R+nDZlfvFijUIqabEq6vPZRERFRl++vvnh3T0scnP7Hib4+NkHb9697c9TntNxmrGkLBMuetnQBPF+d4YIqHnmnJJMBgCUsmacl8ulHhXmCCLMJEGEpkTb48g45iIjJGLJvI0RwHxZcEygjyiBCEDw3gQ/9JObfXu2wCVZiGVT8qy4fn7Vj91wbt+9e9jd9S68udpWUbv+DLDwTITrx9dVVYEE2kkbF+8fDtEbTElGMQA4pmisGfc6r7PE5kk5hLmL/tHq4t//33/hffy7v//HYRqiTykERlhKOAWUZ2XwWhCg55Fgjjlv1jVvRAAmIC8kC56dD2cmOCvwullykjKMg09T3ysbhZCI8llHCFM3KcAJ5HK3b4PWCHI1z9MwV3VZViUV3GrtQcIAJ0ydS3p2FMN5mvWorbEEY6Ucxml/dyQgxRATSsjHbug9CqvNhlE26XkYnTGTUQOhadeCHw775vFWiqybp76fyfriJ1fP1kVjTlYErA7n9bJJKWaYAQOiRUiH08PeI3xztz87HzFJxgeUfnh3Txi/feg+//qVKErtBjcO2ph/92/+L199+eX59pwEm5E67U+8oLMF83lCeT1OZ+cNhYAjnOdVXhUJpOU0jN3wN3/2XxKIhCSIYgLvMcfWW2BNKSkjUBACUpqHHscIgx+G/uV36rtvfrBWzWZOKFGCERXWWRfidOhjiELqos4x4TBEFECw1jkwdppL4ZQN0GaSaRe6ySdIHEXLzXJZSQGxNu7c9u35rKx/9vwa4ERefPLkq19/P2o1DpMoZIDIRdvd74WQPEmEcFkW3rlSZpyViBEIAZcyz7N5UvPUWztP87BYrJpykRdZjDFCoKZJ5HlZ14liIZ5RzjBBKYZ+aFFCHOfjOL26u8ME/fbv/t7Hzz6klHz11Tfvv/78fD4eT0fnnVYKY/j4g+enw3vJSgAR5oIR/tEnzwAGalIQSooRAIQKdG5PX335RUhOZhSHqJXePewoZRdXV1Pbd6deMA5wGge1iYlyNkydMjq6sFysHz9+iiiu7t8fR7WtlhcXW0jg66++V0otN5cJJDXPEYNj11utjZ6HsUcAT8YSSqumdkp3/UA4B0YrEyLwNBfnvlus14SThGKK/tH18/cPuxTpxfUKITJOXTscxmEYhh7MqWkqRnNAeQD00ZMngtZ12dze3ofoOKUYLbrxlFcV4jQl0B6OxvtlWUKMtNOQEMblxeOnLC/aU++0A27Wyr9v96zgsii2zWVEvmxybdQ4tA83N6Ku6mbtcMScQRitV95765x3PlqPKSOcgRQZFQQgEKHMxTCOECAIcPAGx8ioYBgDaEIITumRtYTQAJPMZHAxE8S7YK2lhDMhMMHTPIbZae3rsk4hGaPLKrPWOelCiiYZlgteZDCBEFyMEUG6vbhEBNvMWKMjiCIXy0pUNoU5AhvqurrYkJu3t0rrsighJlVdggROu/3d4Xw8DkxiDFQhJKNMLKiEBLpgtPfORmCMH6UQlngEAQbgeLyDinfvzUM72OgiQpPWEAFEs6LIACMkOaMtADDA4LSOKHHOIoIhAYIRy3KrjdW6KiqcCSHzGFLftcqM1sIIwboqAkycERyCtwaiwAlLCcKUECVFWUQbk/GcMxN1XuSPnz2rq+byciMl//zrL/pjb5Tydvz+7dHT+Dv/9cfLYv3s6YtowWF3HPreT5OeeqUVQoBxihC2zoQYYvIJpOi9UQYijGHMy6xeVL/7e7+Y+xlEjBH68tefwRgBSt55njE9jz54yghEWFt79Xizud6yknngAnJOzwm5kJxR3kN4uk/ryxWFgHGiMMAICsIRIimEoTsHYxHC3XmIMU1djxHwCAAMIcaMZ1SQEPzldsspd8ZEH7w1iAJOSV2WYrmeZ3U7vu/acewmIaWgDBICILRKIYExAhDiEPQ4T96ZaTQJgSwHMCX7bp/lmVURKusLOw1jc/EYJY9gYhAZpQBh3hiYkJ2S8b5Y1l3bM4GR1RDAuTsChGSWlZIZ7Q+3t6Maiiq7fvoME348HPKivLh+/PT508GZ07OnZzV99+olwcidTzTH0HuWV02zIlhQQiAE69VmniYELIgRJggTDMlKUcHZj94Za6CLLFHGs2W22hYrDGF03tpJm5kxwRbcBhdTpEzkJK9rNJfKGYswCMHWZd6fe4xAxuRkg591NNb4GWDXrHJRZBYCbSyWVARGIGUIEOxJQTOcOZfqjL98/QOxyi6WVe4Lo3VEqamrSc2Hh1ZInMmyKJquPd2/f9tn/axMluWZzLjkMCIYYUpIKX04nhGWl9vrqqj6sVfznDBQWp8P53JZwBgwY3lWRoeKXKKEvPd9ez4f99ePH3/86Nm2Wv/qq1//8z/+0+3du77rAYQJgRAi5fzYnmbYP3/6YYh+HgcpMCQo+MAZ1drEBKp84ax11tze3WqtHj99ejju7+/uM8lLwRHBqhuqslzUBZWkrDNM0Nvv3zrgCWFNU/noz6cDFVwyWmS0KhgEPnjUa80o+fDjj2wwn/3yKyGpmsfgXIrx5u0txDSLine truncated
|
||||
"text/plain": [
|
||||
"<PIL.Image.Image image mode=RGB size=512x512>"
|
||||
]
|
||||
},
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"pipe = StableDiffusionPipeline.from_pretrained(MODEL_NAME, torch_dtype=torch.float16).to(\"cuda\")\n",
|
||||
"\n",
|
||||
"prompt = \"a photo of sks dog\"\n",
|
||||
"image = pipe(prompt, num_inference_steps=50, guidance_scale=7.5).images[0]\n",
|
||||
"image"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"id": "2efc13f3",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"{'pet_config': {'pet_type': 'LORA', 'task_type': None, 'inference_mode': True, 'r': 16, 'target_modules': ['to_q', 'to_v', 'query', 'value'], 'lora_alpha': 27, 'lora_dropout': 0.0, 'merge_weights': False, 'fan_in_fan_out': False, 'enable_lora': None, 'bias': 'none'}, 'text_encoder_pet_config': {'pet_type': 'LORA', 'task_type': None, 'inference_mode': True, 'r': 16, 'target_modules': ['q_proj', 'v_proj'], 'lora_alpha': 17, 'lora_dropout': 0.0, 'merge_weights': False, 'fan_in_fan_out': False, 'enable_lora': None, 'bias': 'none'}}\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"def load_and_set_lora_ckpt(pipe, ckpt_dir, instance_prompt, device, dtype):\n",
|
||||
" with open(f\"{ckpt_dir}{instance_prompt}_lora_config.json\", \"r\") as f:\n",
|
||||
" lora_config = json.load(f)\n",
|
||||
" print(lora_config)\n",
|
||||
" \n",
|
||||
" checkpoint = f\"{ckpt_dir}{instance_prompt}_lora.pt\"\n",
|
||||
" lora_checkpoint_sd = torch.load(checkpoint)\n",
|
||||
" unet_lora_ds = {k:v for k,v in lora_checkpoint_sd.items() if \"text_encoder_\" not in k}\n",
|
||||
" text_encoder_lora_ds = {k.replace(\"text_encoder_\", \"\"):v for k,v in lora_checkpoint_sd.items() if \"text_encoder_\" in k}\n",
|
||||
" \n",
|
||||
" unet_config = LoRAConfig(**lora_config[\"pet_config\"])\n",
|
||||
" pipe.unet = LoRAModel(unet_config, pipe.unet)\n",
|
||||
" set_pet_model_state_dict(pipe.unet, unet_lora_ds) \n",
|
||||
" \n",
|
||||
" if \"text_encoder_pet_config\" in lora_config:\n",
|
||||
" text_encoder_config = LoRAConfig(**lora_config[\"text_encoder_pet_config\"])\n",
|
||||
" pipe.text_encoder = LoRAModel(text_encoder_config, pipe.text_encoder)\n",
|
||||
" set_pet_model_state_dict(pipe.text_encoder, text_encoder_lora_ds)\n",
|
||||
" \n",
|
||||
" if dtype in (torch.float16, torch.bfloat16):\n",
|
||||
" pipe.unet.half()\n",
|
||||
" pipe.text_encoder.half()\n",
|
||||
" \n",
|
||||
" pipe.to(device) \n",
|
||||
" return pipe\n",
|
||||
" \n",
|
||||
"pipe = load_and_set_lora_ckpt(pipe, ckpt_dir, INSTANCE_PROMPT, \"cuda\", torch.float16)\n",
|
||||
"\n",
|
||||
" "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"id": "e50da571",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "b405066c203e4c7fa45a0c3389f76796",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/50 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAIAAAB7GkOtAAEAAElEQVR4nGz925JkSZIkiDGL6DnHzN0jI+vWM90D7NIs0T7iC0BLeMdX4xtABOAFuzvo6UtNXfIa4W5mR0UYDyJqkQNCFXV1ZoS7XfSoirCwsLDy//Z//c+fbr+Oc4DAmT4QESRBpuCo/ygFwAAIMEoARIKCsH4GAGBSgiQShEQnMwGAAAhBBAFIICklCMIkkRAAQZCR6tek6nfrJSQA7B8FABKCBBqRghkB1U/1/2l9NkICAeL54s+XhURQhOrnhN+8Qv8k///+Yn9K8tuf11usNcr6yqovIa5PxPUdjUipvxS/vV19WpBQAhJZn400QVKSLoEU1GtDIAESUIK1dN9e0UitP/r2WfHbNVY95fVRUkK9DgRj7wSjpWTrLSUBVh+ivjZAqr8dobXy7GcPJWmoX1jPtL5rr6eItWD9/X6zdIRSpEkgs/4412Nlr3m9kQmy/oTPp1k7RDATANVmk9b34vPd613++7emmAAp60/57TlSEm198NqXqr8l6yMQer6U+lsQz90swelTsloGKnvdCMkMz9VdD6t3B/ncb/VI1geTAK1P2Gu79u7zXEGS0b6daih7m4GAEUlBMHpKRmH9l7REb+4+mb/Zw1z7SfWQqfrgqsPfz1vqnyJQ65fql6zt3J+aYKp/RpTA3pIQACMiRRD9Rb6dYkGs8NKxoX+JtZySrY9V/1wn57dnZC3v86jouSexohkggik8I4GEPtPEigLPAy2i9nC/U8UxVHzqf6+naOsT14d+HuH+hfqzZ3CrQPJc8jp3kICsBUwjlBo2iS+//x3/7/+Xf7Q8U2SmZb1MfQkINKgCeh/g3n1r8dab1c7PFGH9+SEYMkGQHbHVR/cZgXqD9lYhe2esM9pvv9aCSNH57Q8qonWws4obZsxMADRmwiu4U0oZeyn6YAhmlASqQtgKfH188P+z7lmRgqQgwpCRNAJUyo0pgSKZWa/VL76SSOWIziwp2HNFn6dZNEJrs2TCSFqdiPqgtLXgHRuFCnAkMkCSHTBX+lj7SSvHrKWCUv35a98rbYXPTsCUMmsLWu1rKhPfXqfipVY4WV/nN5tohew6rBXtlOsp95ch8VzhejrrZ1aoE2qR1SsowKQK7uhAuWBBnREBlIG12CCZ9QrCc4Om4KQUzxSFtTv6f59b4DcvTT7DLrIgxTOxgBK8zwIE9IZk/65Ee65SHe+sJ5g0ZDbIoFU0qE9S6bagVf1xvV9ShEH18sbMrAishNc/9NmkEmYd0SSRlimz2mS1H1gnYr1vfTa5mRSokwJD4b+s844OneuZ1yLU8ZRU5+v5E7S1zdVhkXV6UrRK5/WriT7Tv0Vg/VBJg2UKdM6AmT3xT2blJAI0MDLMmRITrONZ71xPYL0kBKsvkOKKvVhQsH/8N7jvuWl7Jb/t4c5mbsjKilnncR1YrpDyjHlEPZdvsJO1q+unnqDhW6LrP7JvWb++ba41VR/erEjXCSkruqbAOvIpmeO+Y/ickA2rZCvSlQmDkSkh0t0KhpghUwrRGoTSmCmE3FkBjkAKiKSTBR1IgBlJwtwSWchCFW46dBCgIszsuZogFAJpZloptzB6L6u+RRJj4TJTJAVaZx2kADktkUbLFKu86HNS+8H6XAoKGQmDwfotUzSSzPVGHV0SBnYyoylVyYFuhXTVO89gVKYZM6FMM6t9bmT268PMMutYJtUPiQZIiEqiKMSqVEX5qlkSxEwzkoSJCxqBNGcmUOe8ExFFINYpAYgKuFDKwc6pAFL9L7QVU0SAbgWvBWCq/joLY0qYoPMbZAcVacYKHwbWNjWyvqmTKVHrYxRMqC8IGj0yKdEMbCTPBlWGeogRlZWdJJESErTCAar/RaLWp36CHXvQX6x2ptUmB0DNdLdGjnUUQzTSLFO9aWcYaWbfKrtUvw5S0WtOLMQj1JEBqez6h6DRgon1897bmphpbgJF9WYPGb8lg5QQrHjdqSZgIp19SlJkAwqg8oQKf7KWzwpek2BCXrVOJMRezV7h7FO5gLBm1qc1rmeasmENOlawtEZ+DS/qcNURsmetn7WFQNS7V4b0gg8G1q8IMq9cjhQ4KyhwABAzEgLdhjEinSYhI0kjzCmRGUGz2iFmHhF1AJ0MQZnWGAQgjBYZRiOZSopKVRQyMrLjqZklVvJp9MYENNMImmU9cLFKPVaqykVvFJYsFCXQqthigkhU2oZqZwoVugu51DZGbwZ01DKtECyBlVMBW5uvOBw3kxApN88E7w8DU6hqlKClSUoI39KpdTmbEgBzNmauJN/BiZlS8ztpo35AnZaskn990ypc9MSG2T9VAal/K7OOZQU6SQUTGsaqy1yZdQVWJ1/KRmcLKFZhkcp69wYYWanuiccaGxRU/cYv1eehpP7uXRtI9lssk/3KZsxat5XeCnc814rs3J4pNuIWDdJzZbLDRmVfAhAdoLJRf1WO9W3wRKP1muh4UadQWSGgqYisj1654NvnT9SHW+v2JEt6tfvZdLqguhhRnWwS0ShKhUS0ypOCfvWAlEljKGsnVPonCrE2elzvm2aNfFPJ/it1+dhPv4CcWPvb+tebvamiNYX/bn8is8gX0VEbXl2xVdGW60PoCdmkJ4tQK5jNLTZ0/bav0ClS9Sf9EE0rEK9tvT7MigDIjGf9oD7Nsm+pUFLvBFr9Wi7g2XRKVTf9samUsmim3snfYvITg9fa9ir36rPrMMhGwR4Joi0+KdceBowikJkCI+NZfPSTyiRZuFzP/6OE/O8WFsgM6zxbz7eyNwgwYQU9iMIH9XyqiixSI2bVQ0mqIGVCME5lKm0zQZGZ9eUAEUmmFBkg4IzahQYjYBXZoJAgM0dvSVbQq8Aa2TGHZEV/pTo+/vY8GiPzSfg02O8d1ZDheUbWmqtev8+p9SN7PheuyuG5eVMy68xRj7cewjr1qwxdjw20WkUOq3jiSMt6Kg3uAwBHrT2ETGbVlUj1sytcgkJslBIGUF4fuYIlQWOddhIxAxRMtlCBMmsvkrCGPdGPdv1WHblaJxa5WhyiiZAyn/uGpK1XqxcxXyUCVKUorQkfK3QDZvQnr8hWrw9T9TUyQpkq+GwwYz11ZEIpyoykzAhlVzIGmsyq2MqEYKKBtoKyEhS9ir4sWoRW9Qqa9WQdvAJ3KaUg2qId6uhDZs5Kfplg9spQqjxsIODdFQGejwPIiFpQGliZYBWaKERU2QVRu7APbUVqimCtLZVwCHLr6NPtnHWiaACzCiU8nxc75poZkF1BGemF7mpjr0UzAsl13oCsPUmTjYWuC2BXLq+8YJWM0RW5Esxv68lc36sSbnasBOl13Iu9ghnMGl2Ygwa3ZlEUU8iKT1IWaDCnOd2LlMo+oBk2OuPSKSgzaOjnzsL8/Zk7O1JSyKIgYxPXKVgCld+/PW46jb1PQNVGMgIRtZEqeLH2CYqSqK9m0LNwLJyMzCiogUXA1zml1a+wsRYFQ+Wj+hj1ecxEVn6tTUtYP7J6ymJWU7EzsVkWLKqzEeptYJAxpYjoWs25oMMqKFM+6A5QoSkTHDIkg4NJBQIDvplMqfC9olOaV/1JGs0tgchIQ1ZqNXJYJadcVGFvyuLfJK1uZZGrZgRF6yQBZX3f5nKhRqWAGY0yR/+rs85FvfwzRnXDrxLDt9ds9CCsANJlnjIC1vQ/VraoQ2GFPDKFlEkUjKnaCYLCYWPAo8sAkDRZFOVUyAWGZArmo795Co1DrI5LMVlV0kCkLxyYDT3MvD5sBLp3Z964fZFbpEuh5G/4sdqEi5RVrxdgzdM13w0pkR3WuTJDHXCaIQsP14tZQzrRzBcNJ4Go/gdqnUTrkCklaRHdsWE98Kg6xrqOBwFEJMAsEOlO2op6ReTDiEx1viDMrLJAYcKO0EVjRGME87FQE0hYV1LIBsMsphGFyDPNvF4tlZm1ZSgJ3mQ53SUhbYHZisESiJCcXWP4xooECXOr1SzMXqtXH0jV3qdDYoXq1OJqVgOn4QcWnpNUJKlTqLoNnQIBNyQVkCrwe/8KWPuNhDJVZTVIG0qR3n+lhDpPPhu4KSKb0aV51yhFLLFbVilRhq66DMnqcdNMmfWYvjXhzbGqXaN3+SWDMQr8eCUTWqoLRYkyYjGNasy4qHfvaF1VoglpSEgG0mhC/IYsHaYCn8zousl9ZDMqEODuBV+rEkJWfuuiu2KZ0VT/nCiej3AhnSMZWQyneX3I7K5C00kZoLmU9lzArCLSIHWdQlaTgrTqTgmGbBaoj1sflifZ35FWgC1qvEjC5jcCKKS5V2iDc3BYYduYGtv23NQAaz3onilzX3VYMUJEAkazQQqyRdZJgrkXbwNCMxe9DSvs0IxAV3lcLV5JbKlEt2XZxA374MOKgUx0nu2mSwNdwMHK6VrleKER6zYXBCWbriQhmTmyevcV7zvqd4VY4XcpMpQ0elfsRlCmqBRBcvH+yspLlScL7WemqlwlADY5k5VzKgcWfOqP1fwfoeiTutA9VyldEG91JyoKeNHf+vYDz21f+6UYw7VoxRJ0UdrVbWFwrgZ015AwPqE0AGWAyMxF3XJxEYLJvLsUK/Y+1QqLgKqXcq+8lBlCundxUQmtFhDWW6cKfDaulCphsLdU/0BIkpHWDVOLTEnmTmughCYuSEFZON2qi1DnRJAy3fuNuV4OvZ06wdafr23Wwa0q6SZ/itT01VBqBqlfCkYUs8dV+i7yqtZSmaDMjH0mUAwJQdpzUVRBfu2TlQUN5lZCoqyGWvaWaKIZ3X6sHpyaTkl7ljL156p4bHWmQEYmWemZT8JRfIYbFTnQUoZEVlnjLH4A6FpQ3QDL5+rRn/mto1yvOIQu8oRFrUBpNuoE1kMUZEVvsfIZRdiomjNrV6zIVlVelyCV1yNzFVJVnjXPYg2LoF5CFc4EamGbMu3lglIITcBJpqRMVkRnsyckYAaTdV0RqLdxK5TLine truncated
|
||||
"text/plain": [
|
||||
"<PIL.Image.Image image mode=RGB size=512x512>"
|
||||
]
|
||||
},
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"prompt = \"a photo of sks dog\"\n",
|
||||
"negative_prompt = \"low quality, blurry, unfinished\"\n",
|
||||
"image = pipe(prompt, num_inference_steps=50, guidance_scale=7, negative_prompt=negative_prompt).images[0]\n",
|
||||
"image"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 24,
|
||||
"id": "53af98c2",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "ea05e638bda4403ca91d699609909f40",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/50 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAIAAAB7GkOtAAEAAElEQVR4nGT9y5ZkS5Ilhu0tosfM3SPivvJZmZVZVdlAowmgQSwSxARDcsDFEdfigFMuDPmBHHDAAWacECRALva7u6qyKh837yvCX2bnqAgHW0TNE7zdnX1vhLvZOaqiIlu2bBHl//F//3/K2BCAORKZIIlMEAAzkyQAABlJMwCZaWYZATCRRmYmSGQm6h8CiSSRqV9ffwN9AmGZQTL7W5HITH1bZtIMmUA9DGmRSf1cfVeSlqEPqW9I6MEBGjIAA5IEE5GgHpskkJlgv+7tQ0gC9fN6hv7QXpP1Ovoz/X+EJVIrlki9PmgRofWkWUaCRPQ7IkmLmGaWGaABSJCZ0KtlEARRu6BP/bN/uFYMZAZIZiRNX5/Uv/Wv1QND7+laf2T21+pLJ2n6Rr0RMkmQjKwnYO9xZhKshc/6EC1iRpC2trs/n3qpRBKMfsFarv5Q6FWJBJGhB6c+Nqkn6IcDyPqDTIBrb8HbLpEELfP2Or0egCyqlkifB2TSDRGA6XdIRCTMEdm/FDSLqJ9/Y5PMN7uU9eM6I7cNqL+97UwtVJ2mP3+uMnUtdAbfnM3+X4A6U3q7TPDP7VbfXMenH4Zv/pKoNdT5QB/qN2uTOntlBYm2TKzNwO3Zy5z0CNqCzEyj16G4/RqBRCTtzbu37fXa3Ax+rX+dldpmfU/SGJE3OyCRTARzvULyjcuSzWRmLx1vL/XnR792+M/2dy3ROqQBQMbWPgHLp731HmjfaKSOf71Lyloi0U6jFjzrZ7Let4yHjEj2x/bfJwGWzdf3lkPISAYAwG2bg3GytIxliTSUK9TxtvXCco2Jcrug/qT9IDNha50yy5H2oUMmqbXWuZI3RwSWocCYESyHRKSWRr4ZDhkR2mwtE0yr7XwTrFhRp740Z5pZBaik0SLD9IORpCVMzkbrhChHgkAmaTcT15qsg7F+xowp2wLlsnQsHcyQF9f/vtk4EIDTIZ+5TCNRX9kWRRjXkQAUSOpPyIxMQKGKIKze00AkIkBbh6gXpP6QTIbCeab2XesZvX31OAkknFweyvQzKyLX69YKAEwYYRGTJMEIGAyAQcYDEJ5GY0ZkphkjdMwMyApOSCZpBkRWQOjNT7Aicu06Ye2EFdV7QWkIwHoX2J5RJySynZFWyAgAwdDhlpHo00EgEplabTJhaLdb4f8WKRd06bOAjFw+Zx3j8q3rSKe2QXgISMCIAG1FYosVBsFIsPHBctd1ErF8k2yVlLOIrLCb8hc3qMCCfym7Rwr2lStY0eK2gGhHvh5VDxOppcxyf6x9gQFpyQ7/wmlBstZE52uFVIDJQooVTArltAfO2zPLKgL6/MJPFVblxhSBuE5cBMzQ37scvSEzc7nChhe1PgU1yqHXQ4IkImSPEbLb+nWtsS0w1+il/rZfuYNF4q0N36CM1QvKqeokAgAsCSKj4rM+x4wZivAV0gTLEOnGSIAxL3OkntnlFZPo+Kl3hZU3vQETLFcOICJBmHsidHaEf2kCFHD3iCn8WidN3icJJnW6sT44jQZ9awVqOeqGqmjHpJMQAbfMAPQMmaGdM1aIKVQFtiN3yAGZWSJhFcBMLwxlNuj8ow5eRppZ8g2yqKeyjFlOAzA6FqKWI8lJ3kJGI5F6HmQlJDRv0wma6QSad5jRurF8BI23FZhB02HPghIJI5MwWESYaa8YtdOWmeZWaEg/IPdpFjIXpJF0U7CRx9eeaqlZwdpAJAtPNMpbQZjQwTO2lWN94MKREQHQjSBhlTo0WCU7wahwoP2gkqvONtqXZxZ+NWtQKQClH1ZcMhT8NLDCP2uDsj1zBMux0cxiBqn0zOr7wd5AyxBSk2dK65xMj3tzYctoGw9ioex+ftQGIUkrCAk9X1r7ZmRGCmZpaUxvJTxPazzC5Xf6q1amCzNWLmeGsj92sOgsXIBC9q9nwC046OexXNPtK5hubxPDTkdg1tgMSa+IUp9jWGFqORYKoTSoUjBZUB06v0x0Btb7peTS3u5sriin7SMT0E65K0XWm0Wd7sjMcPmH8jzZS0gg6Q3M2bmOgDXLRVgoAStk1puP8uAGJQc0yxnLMVauUevZAVbeoE9WFu2SEGUgL+qs5/Fa4orRvZjJzkGRdIDwYADmbjTSsp8WsnVap++MvMW+tikrLJy9Wbl8ZXlT4GZOocyh/0lzHYyoaKHDaYUmRYRkIblaRxCRQetfvEXJdYgqGNIol1cfoOS9grRyh2R5v4q4bw36TXZW3yLzMWOW7XUSrBXAjfeo/1MfFQWH65wiIs2ozym7Fly0SnYyM5k01gY21qpEqP4wtURyupHZvFHeLNIKSAYT+mFEIBRUQIHHzIje5UxmivIiYDBW2EP5lmxGoXLMyGiUJneDQibGjn8JA/WmeOMysrZ77Vohv16M9TlGGuFWtmPtaZZvvUHL+vw2Xco+VzoC3lZPX4PKMZDmglNvnWatYQUeWZ2VvSUBJi3bRGdaW6xQIVHM57JPAaEMaDWox4hcPEI/VT1YedWoHKdsuFJXIFmnT1at/8x+higDaKxGwYV6DKJjV4PIWzKlJKwinx7p7W4WJMaCxHrNOim1FyFMUSht0SysCFXZTR3Mdcyjn6FJUH0dohawkosEs6FBsj2lfE57rg4OKCCcdUYI03t12lPpU4cddDqpZ7PUISqPob/qGL/2TRapz2/zKCYiy6uAyyEQsEpidPyw1r+ytD4jtQW9SnWIoMCEZWxlY1FL1p8MAAYZvLxlH9t1fiCIbEYmRsV3c5l3ROVlUPxd+Aodi5pd0UkudNphFplmxkbPOkRGZRzCL83ImrG5YJM9VUp1Q/vC6VGfVXS8mZOAzMYMmYWA2XEDIKKfWU4swSZ23sQqdnIUUXi8l4g0RqbyiMLeBbuho1chWQsabeXWSZlVENDzr2fLTLNO5cgIYRkm4LcFK09qtLzx8A1QsVJRIGDmda7cEomZcDMiIit3ywXVa9tSmY8yRIFcKLhSS92gFB0iaV4AKBVprGgicUdYBBYrr9Khq2WzhiSZCdSSEnXAG/E2F1cItGJ1ewrh/nJOjctMZAjrRQp3vgkVHTCa/e+SVcOrymxE35FwY+JWs1lGCBoQJOXr7A1o0BFJJmgVMCvZyCr5CK4WtrzlB/1gASC9YoKZ41ZqqR8zWiIykRmEPBGKMZ/rhK6DSFD7w8xbQaBcANlpQdHcpEUXpaIgQmVeJIKVbQq1g8hbUa05Tza/0rjVzCI63RSUsn42bbgOYPS/g5V6st20/LD+3W7Un7X/ysKCKjWx05q2WaYWjmznGAlhiuhz15xzYQMyEJ1hLPspJJki/eTvrANPssOT/KCRVV2wN3gUK4RZLrqjwZqc06JQE7leG+2X+nTcQHTl/MKeqIzmxhbIh2NZA1lRgE3Q9QrAOApNtZuxTtWyWeA+270u1sYEAGllw3UkCv6+DRy31cwE6GCn3ihat+qiLGKVzXEXCBN7H8IfVlRlm0XvUTssWjPe2roAmZUYekWU9cJ1vMHGPt4lB/65tQZ6TUAaDRZ4Q0SayZtU7ntjhAAw3hRR0o1yoPoB75/UwYOgQwHIqhUo8DFJL7+V/dI0AGHGaCtzd/2tGW4uUrtb7x1Vrqhfx42hoB4YXZ2RW3tTMJTJq7JluXL8zM5GU0ebtyoKUfUeluvESksrtnoZt2U7FFlQdtIvRot1bmQFbf/li40AbBiiDygWhfzWiG/FvXq4+qtG0NT/sLxRUxM6/NlexsCoQgtR5YV+qQyrErxyu1Wr1hKnrdgTCcQNbgMu6rkPf+HchFJeonj/EDtvTB2N2oBeHsSNbmjyFgDLcRPrsBTnFFbFjTprDfVRDhd9/LVCXIlXWUshP2CtLXKu7WLC3NCxWc9ToKHqbgAX24n1M7WzteH1Hmx80P+Uv5KHLW9QFFT9t2oYBV6jlqZAgFLRAl1Vl0IjVazAWjGXN2h6S4YKkXSFkXj7IV3uW5BjfWo5ytDBLnsQJijHHoUbBDKyyzA34+kA3gZdi5eLn+8sUAevt48dsmS0OWiiGFNHuG0DZl5rnYJZMEMGcnbZRTts8tvpFVXSvLwbSdUAbzlXVIJfNAWqkk9b1XwFoZYGlZ2W/SLFxUvusjKSsjOSsJEZlXcupF8flW4uHhzo4navUmbV4bGY9pUsRdK9uPUqyhfDaQ1IMkBHVVE64dIW6uttVfMTbl4l1trmldsgZ9LLJbSVL8MpV1YQAbkIp7I7Kh3VJztylknVh0SBu3WEcsV4U8FMSRgrC6nsp2oAymvBjBABWOQEla/I5WVh7dal3HCQxEXAqsuhHLcZFA7TaBGzDaeWpKup7EWlie6UBQjyVGCu/9MHQOaZtxVsUcLyjwaLnIVYSuFWYrAiFd7UN0BYFfeQQvztuNmgGssOy0Pe/NSbVCOp+me77oScLxJh5p0Hhw6LEjUFbHOdU1sWyDdfU+m+dfyMpLMiszXLtv7hWhnREaw0vVI0vHGyKDyWGdF8PVbildYYr2Et2mJv0aZQUTQLh+Xbi15aATl7t2r/GtWrpoi2OrahLBtYrreOoFW4ci9lFBUcACDilhPV89R6ZucrNMXyDvONeEUp98kqNku70janp5JPWLosfaEZACvWa0GDKrChQG6jXsGhxovlp2R8ZlTg0lcplEWvs/ZlBew31okVIYTRIzAqvtbBL8OWNoON0itP0pN5E/xZBq2dEMozc3TSl+UIElX3QCoQFd8SmaJKKgFH403CgFkWxDo/stpOUNgHD2vRSSSiAGN24izHZMZEZBQskCcqBYiVCzUAyUCn/oK80jWEkTREZ0iRYdQehGCIkNrCQLjB1LTSqgJABrJgINnVGwkEkGGjTkUH9soKWcl05XNy9YB8VkX1Sgu0TGvvGk8u3FQhYGYl7rKVUDCTHaX5aFsrPI+srDNKFbbARXZ+XUkelHbMydryyOUOKhOLine truncated
|
||||
"text/plain": [
|
||||
"<PIL.Image.Image image mode=RGB size=512x512>"
|
||||
]
|
||||
},
|
||||
"execution_count": 24,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"prompt = \"sks dog in a steampunk city, 4K render, trending on artstation, masterpiece\"\n",
|
||||
"image = pipe(prompt, num_inference_steps=50, guidance_scale=7.5, negative_prompt=negative_prompt).images[0]\n",
|
||||
"image"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"id": "200d3358",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "b669f753a13244d1884b7956ad0ccaf4",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/50 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAIAAAB7GkOtAAEAAElEQVR4nGz9V5NtSZYmhq3lWxx9Tuir82ZmVWVV9TR7ZhrADAcEYQDNCPIBr8Bf418gzUA+wYykGTVIzrBn0N013TUlMiszr44b8sTRW/nigy+1IyvyZsQRe7tY4lvK3Tf+t/+b38ZwVGUZIBIBQEAEIogABBQAiYCQIP0QICABAREBAhACACIAEAEgEhEAIQIRyT2QriG5BogAARGJWwNEREIE+TBABxApBADAjGIGoWjbELAJzd3/7OX2f/qrIazj//X/9vv/3X/3/2nf/9svT7aTbLjDddzuTyezb379zWAwWtGkpWy3/jwYFNOj43xQTOsq7K9ng9jW2zyHF8+/GC4G2wY+7/BPf3pzc7/BvNjXZRVxu63OLn52e/cmtpsM4tOLZ6fH87PzixfPz3dV+PbN+ts//e3DXfP0L3791//5f5pPF//4qf2wyrvxSZflkGU5xoAYKAQEAiQgBAQgIIBEMkJQ4hAQvyf9AAEQEAAIiABR6Gw0BAL9S3I1ECICQJcuQgKCgMIvZAYDAiAQQWqZGQSQOIbCoNQOYiCKABAwEBASAk8GABAQYyRAQEhSwgzmdqRJnSwBxEj8ZRqtExRMIwKKUeZGfGkiAzeUxiA0ELpR6pgiECIiIN9BfoosqyKV9gGl3sExA5WuiWPpPcmnrAPaHqD2E5gGemsiFXUxjSmpWUj9kicQQiSeS+J9om1qVQUIRakAkSIxp7mFRFIMqJeCpwAG5n3SVi9XLD2k0snigiIPQjcSSaM+C1AQAKwBZM5Sb5YibgwflKhH/qrIVECWE+Ohcp+UiwSJnMouQOuOZwiASVABASCqRJGOifUFHmFX4qyImV1Pdie4eTqCpO7AUYTxz96KelNMwyQACIYKhDx6ZKjAQGQSwDwFUNUWJVaYEJHVwVCHREQBCae4yw9hHqiADgEjJCYgOmHQGbHwqaihkoWIWOkpyZYAmDA03cXERmKQIwAknmlIo0zymJpCigAhJj2DGKkLAQIWLWVI4RBhReM2zB8emu+u/hAIYnc9DpPs+Yv378Z1LN5cb6GY19WqyeHs1XnWDebVct5tX58dxXo7G5Z0Qm2x/+Hj8u++v7v6vFptDgQLyo5i3m3Wy111s1tvPl9+l2Hcrma3J3h5C9/+8bprj99e3223t/nkeFTMJqNJWQwzqFoaRygBEChEwkA8757UiMCQwjZ/QwLrfBEpUoPqPxARBBV8wQ5krELtCDEBlHJDpJjkJr5NUSqhAVv5KIMQqEdKnEw3EpKT6ECBAAiiIr30qnrBkCAywBqbcM6phkgTozBPJ0K6P7Wc7hHo59vEKuiYk8+SoNxcF6EhQFAxZKOlX7EJkT5Mw9xv15sBrwNm0wBDfxC1FisDfA0TzYOJ2k0UUBJLQAKDpGwCmZ7cQQnM0y8Qd0G0ioDYwgjpUM1MutchnzA8CWQU+UwEFyE1BGXeeCIBysiIvH0EljgH+OkVYwYJOwIjEtMiCmRHvh+F40CA0WjIMiTWgJETCAkQg8lhdOiMPPbovbM0ciIwfBMJFBxHlhznxjBvZPLadE+EWNuZzF6miceQhBEBIQqcAzixNEnqU1y1KF0axZAkkIjJk0jtb5txTpTVgCEQQkCKqZuY0J0AkhfriOl8AYIICMnhEb/J3AoZbrJhNiHRGgRxkCFdwaYeiRAiUAhA1GVAhBkRIYQQqaAmh3YQw2G7GrWfnx3tVnk1HD8/H46L4quvv/jFFy+f33b1D28/3Nx/37VvOsjW1e7D5R9PFhfDdvvq5LgcDuaDWG/u7lf3z6ZPN3fVzbv3f/jT222zHc9f5sVXMIDDYb9990O9v9/uqnFe3Nw9hGy8Xr2vtsvltrp7uB+Oyy//+S9Hx6cnsxFCmOYwCF0dIGIGQAERk8CpDkIUUUTBBhR8YOU2L4cVC9kZV78bkzCwQKWwIl2DLNIEgBggOfzpkkRv7czET3SE9ct8KpQLyIRYxpPgyW5O/IqEGBQ0SdsXYUFKzEYIyacgiffUHoC5tSjYla5HRCL1/iHFNKTXCyWS2QgYgWTI2AMIQHaMg2E9ECIrb0JPRGTfkESz1RqpipExEA0/0V3B83B4iIghyQKjl3AUEvvELImHiNKExmXiDQCJ1ROZgcR1YNcRQHqQBqUlGZgaJZJbEaJZKrFJIP4pcfPJUSD1MsRcsYQaR5SVRCkEDiIO6gYgeyAq9hgwRgIADEn4JGQlCt72gNEvEUTmJbyWGDZRgCUBCSP3mwxxFAtmeihDZJmU+IeZwg4FSi+etUaxPpYbzif3RaLSJJA8uASDIj8e6wFkzIk+RK5jFMmOFNToofgWMqY0RSIdXkCMRBgAESnmWd6FXLSQdAxIRMFPShxV8SZcoMEJInbe5TKx5RgAIaWSQH0eitx00p8k/wmHIiFGAAgQKSbCRKSIMSJEjEUWigK3Tb0/VMdF9WJax69/8dWL09lokpX5L77++Wj25OrbN5/j4bamw+rm6MkLhMnusFtAN1uc7qH94fO7L84XLxbTtu3uHra3D/sPV9eXh2uAEOttFjdYDyAb58WYELque2iaYrOfjNptvbz+9Lt1vc/CyZOv//Lo1V88+/kv56fnzf4wGTSDIlZIHQIgZCFadGQwSqr1ZDquNDZVcLm1dKvIOpIAgfpIKKGz1wAROLX6wL4zsHyIEDpxMXEWZJBxmY9Caq8cIkpaj2+myGIgAyWAIJDIToC4ROxYCW6gWQ4W9MhKz10LcAXzAUXkGCBICAbi5JguymUEAIHUaxUCk4U+IGELivvJ/FHS91pVD044SzpDNmWiYCAynwaeJq4OOFsd8wpcYofblPSMYBAZJyQFCwxsiT4JC4NMQQUNHg0LNT8oVo/ATR4gmahk/bkZs3ni6JgcWRQkwMpENvvA+CVZKB1Y4jiBZFMoQnJOiYAdByCSeCiNR20A3yvMszRe8uJZJYyiQgWyvBeLJsupuP8o4vSTjC2QzRFA/F/mNmoWjzUPhGyogQSkPKuFwExSE2YvUZxnZWOUFEFMgug56ix4EN5eKfYCYgaQB6DIRhFCEgOCCJLEIG8ZIGheEj2ngNjUkPp0JC8Dj1fiMyQAyFIbjGOSsU3OPyaDT8lpQiJCioCYQZ5ng5YGENf3m4fr+3a/Pz8fnZ//q7N5GanqIjVl/eHTb//+D//h6u7zbvUJwj4fzkfz0yd58fL4ObTvP775zTiHp4thkR+NJ2XIhx2E/W6fKINUA7YRB0/PXr169q8+fPzXq9U9wHpzuNp3JQ6q4WJcbxaT+ZdnT//i6bOfXxw9GzU11NsZNQMijF0IBQJgFxKbLQOUMJFUISUHJj+WHGBFQ8u0JPdLKU8o2KGOCCK7aNxhZPGV3qWj4ESe9Q19KoKAc8cYCQEwJE0D1jNx2ZKfLB6IQamVGAggkKiP1Ags96BCS4hcCICAyfEw+4bitppUp2lH7kVwzGsURZ0ayyW/zAAhCRJQjKCZLKF80kYUlQPBFWMRihE3O8tOECpOqOVG4yYqLsq8QIK2kMKpIPcZkCltJSuaRibGOHmwbCTYGRdGpxuCfMb/ggZRZu0Z+jRDi4QIQXxLNmpitzw5iAMkdjsEDAy8UAXPh1AOZZFcBkc8DgIgisnYoIwvIAbwgTP0pE5zgJKTcSWMNBaGKUx6InaMc1NMIQ07Jd+empTEJyX1s6hN5y0XUIpv1W6Lq6yD06hVEviiyCizZ8jISEkDEDCgjdLw3WgNIbBn6L6R75wXhDoMSQekjnMg0ooBYpSgTq+XocmdIQWsyftIZhkBCIPHMwlBpA1KSY4AQBiZHZzZQ7EwAQj4EuCgMtXzIkBHEbLQBsqK2EW6utndXG9X+/bo4uvJ5HwQuo8f/7jd7pf75ffffXt9c1OMxoOjOXSTrhpNhmdH41lGk3W9v9/c7GPXPj3LMixymByNzi9OTs8WV9tljbHpIGQ4KIvx8WQwucwG69l4vNrctQAwCsdPnpy/eLHflwHGz15cnE6pWl6+21VNQUU2ziECdAE6xCxlsJAwaDaQMTZBqsu+O2aac6AUdPIDopRBruIAANA1xEgYIOGDyIQIbQDxGV1pVfWKICTtEOHTuAHB1I/deQzCXvWLkEuXIvypR9TJJugM6r2k8adrgykLIkWQ4pCSKYBEAJysYpK4iF3GrY5mat4gIsE16Z1C1IABexIPPXdfx6WJB0l8AACH33YtBzVBmKOGilsWby1xFykkjANJMhgD1JKIdEiORcHCZizpAVBbEMDmwLwWDxHtTrko9KQNVUCCQHmSJgVrUvDlsVjvSmzJFVoOUlIpPC6N9uQ36TXKG0AUI4pKPMKUHHHJP6EVex2GqEl2dK6iPpIQR6lbYPKNfdc6MFI57YGc+jvOoEFKcyr46Q8rOjqd1XCE0gzZrEaRa6MQJuVK1l58MRQuinfB/E6/2M6gDospw5eqxuUhc16Wsk0ddpsFcR5HvD0/R0JGOg1QxPaIGVaHlpJcEaqTmxLmKanIniumYDcAdoiIMeShog5i27SHNiKF4eYQP9w8zKZNNig39eF6vS7KydHR6egETg7P7terrp61u7siG+Tj02of39790DWfJ4NFu7wr87wcwHrzsHj1xWAYZovp4nhx9bCsqB1iOxgNzs7n5aCajNvji+l2MxsU0/OTLy7OXiBkm839dHr04vm4LB7q9cM1tGFwVE4WWawgth3FLOXYMGLMRP8kZ2FS3VNBswJEhjFkxNMwkOQlJcyUMNJ4JegXfKKJfR2OhYW95pXq2h7tAVRRUe5hnpC6MKQ+JxAne1mbyHKdfR2HlOsOqrKcqmBRMNRNHZIED4aHplwqswJRNhlkP0t9b8Yu5IAgrZBJBEXEIFkRLvIx1pDHFjYNXFxIEMNUFAnvaToa5QWo0tQDRrZeiVYRAYEi59SB8zeaQhQamn72rQoro5oLine truncated
|
||||
"text/plain": [
|
||||
"<PIL.Image.Image image mode=RGB size=512x512>"
|
||||
]
|
||||
},
|
||||
"execution_count": 17,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"prompt = \"sks dog with Eiffel Tower in the background\"\n",
|
||||
"image = pipe(prompt, num_inference_steps=50, \n",
|
||||
" guidance_scale=7.5, \n",
|
||||
" negative_prompt=negative_prompt).images[0]\n",
|
||||
"image"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"id": "0518421e",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "6e4e70ffd53d46b79955b068403ac229",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/50 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAIAAAB7GkOtAAEAAElEQVR4nGz9WbMtS3IeiH3uEZGZa9p7n+HOdVEjCgUQAEGQ7EaTQpMgWuCsNsm61XqQmR5kMtODTD+gf4PMpBfpVTLTq4a3bjOJbS2aOAkEZwIoAqi5bt177j3T3mvKzIhw10OER+a+wC6rc/faa63MCA+Pzz//wiOSnv4f/t+Rn45OExBAAiKGwEFVASgISgQBWBUEVSJSBQGkAAEKIpQfIi1/hgKA/RlQKJUv1xcKEJU/g9S+Vj5TLkjQdgld/fKlHyrtLC0p7dL1jes3yF4QCIDSly9TrgWlpVt2meX6y0fbp+1uf3YTH7fXmlqsqq3NdgWi1Re0WICUFFqardYHWq5N+mVbP+7Tl1/8GZ+rNsFyVaI2ZF/u/eMOFXupknW+fb18vJjIRkEfX6t96k9dlaialdqf6ovigOU/xaXoT3WpDh5B1Ya7/t38Y7lt9XHS9fjRupnFC0ubHnVgNSDLDGjv02oerNzPbLv+cDH2l61QewvQ4qxmhi+5FVWD0yNzrQa1DfHqS1i9u7p9G8fl29WCpI/Mo9Y8rK5BX5oMaD1WkLWvIIgSqvcqtWFcNUS1Wrx8kJbLKmv7JAFaB49Vbaa02VW6YtNdRUHN/mT+yDYhyQbykYntRfmHVkCzmvF2xaX1jyBGYfZbm9Z+bV97ZLcGRgaVZstlmIBVM1bmVoUSLbDcbkCqgCAr4Fn1Wbr3JzqAVJQdiRSHFihybSpBVOtggFTqIJVRW8amdLD1Rhs8lqlIpBXotPmDTR0QVJa+lyFaQeUjaPnTP4YzFYyhjxtnVra2NCwys9TZsxpF/dPXX/1Chsw2SF9CqcejqPpnXMJcQ9dtIbXXpOuPL2asl/vS/BddMM4M+yhC4BE0fAl7FhReZu9jG7Rrrq6kq/+2LiyfeGSFFWiawdb2UF03dj1QK/SlNdouWF2D9hLWlzBuPrBumCqRTeI6cGro/ci7dAmA9lUU966hbuFGS6cM9ZeWLx4NyJfMSfTo7cqqbAzNH8wOj0PhElOao1gQeWTqGoCIgMdO/WcHcyyAtB7NpaOtee2PurhP/ZA12lpMNjNtfOsfpV5SDC1oNeGbT5TJbJ1Zxi6vmk9EdRwVFfMMfKUOGgFZrEkLR2xOJg2QaD2qK3uVK35peppJHlkQi31q88x7bGZW6Fts1/q1eLvNsIW+mluoqFmXHg0kLYS0+bQsMZqqAVUIUAhhBvReBi/EGY4ABbM2C5HYBCeqJLSMRuuRaA1QqiBw6SJx6wcDAFP9ItncJFKxKVthD2WgCmiXeI46V5vDrWbBn4GCq6i/hHBqdq0uKQR6fB2ihbuiRh5p9i69NTZB1bOoYawqWjYEg7yFDaOO+0L57DcVEEEa4lb/ZetCMWs1QmFLFR+YmjFK81eGKjdfgHBtITJOveDSKjwSFuQqnSofleruNnfsqssEbWCzihO6jIe1x9y5koBikpX968UXarCKWAuPWWUaZPSaWudsXptH2Otqvjrf1OZC/aWO72PAfxSiqitp4VVceUaBHSn08TG9XkXkFtVWkN668GjCN7yo0avMXG5J4DLKSkZwincSVL5M2HUxDcyfrY1KyhVMarPawC2cseF4uRm1YV0S1vJ9VYB1GT2sOm/My4xfvInMAaT0mglK0ryxdostOPAyBxekXOYoyJpUoKYM44KurCUvACsbkVVAjQrYKDTfXo9Y+U8LeCXnLOPagtQqa7cw8ciNzOqLHzOsCXWmqN3lywyszA2FLuTA6BJVvDJSQiCpfNwMr1Au88AwigrISHHfzJI9eWUHrxCuJmYoSb2N42YT5Tr1deW+xgQLVqymJcxjqfrVgkU2pXR9WVj8bbmb2JXJlBG1AbMIuXwLbeqXNM7A/FGwsHSqGLrcC9pmr7Z5V3CJVoGnwkNrz+plvWk1rxhMLjN+8YHVvCCIGQdlDoiAa58UNfdr7VkAorSHa2q8pIVfyphski7/tpFRWLKxzhFL8C5QZLfilkcvQG5xxmTBFQ4XTCFGGx5TVmyw2lQhXTpYP/aYXtbRqV2iNdwvnzGORDX11KUVZm8qRlpGwaYIVh9TMKm2JFLBj1IC84m1M2j74ipYoL5szr6OmdZXNCSvXTCHaI1RgO0XEwBairEwVzLAIYVrLQQEymQGXyeTlsWzMTkzli5tqODVYpwhamUopcHFG4yxLX+EJfqG8lqZhE1OIq1Y3yZ0+cfBeKVCKv9sM6Ra6cuxycZiic3FDjZ0qosZFTUHa/Bib5mVbR6htX5xoWoOXY1ic3LDsTa1mvJYHMm80pT04uhaY/cCp83RGlg140DBlkKSqhBxhbiWHpXpSEWgsWYXgy1RgQiitZd18pAj9U5UEwMgcmAlCBUBqHaGCVCVFRkw17OrNSIMApHxE9j9tTB9Kp5XQqS2b2pjrSWamOnJZNfaJxt/bTRcibkNDGAs79FkI25QWF1J6zKFlvnbZjrZNeo414lsNB+N+6/wffFQsfY2Pyjeb0hlA6XGxGnlJ2XSEEmbtpZktH9qD0EoOWDFQoty1kjmBY3bLAcgDSqbA6zoSsmLFJDyDW6mtn9LpoE67u3zNYSiyaq0Qpfq5FoGoQWrRsGgYnlNmd4txizTwSIzr/IaNe8XeRTlDEXJeBJpZRFqU3oxQqFGBn3VFDARU23gijODiypq0Q9r8FuASVexp8VgXX2m0JpmW7LrlCuz+VgZBou9xaPqugfafes9SElFbEgr0JahWUbM7lVZNRksE5Fo4ynt6/Yf40uqxOYSMI8pyFeTd9QJThZgLbdvMFpTB6rMqk61BrhCUlGPADW9jAz3DcTNJHWwxbC4jGb9mEK5Doea+EBcgpSosaliHqnNbPPoEVjYWKgAzM15VFYqXAscLddfYEOXcaT6hwLUhX7aLKvLHMSG32JTqnReViyhkDSb77Rcw8hGI9uCgm60XnUFE0SVRZWJVdkh+aoBtOEu7sOkqgxVQ8CGdmgYSKBqPJt2DS2wjFbhZ4acBvMr1G1hj1bLXebmavetqE2GnZV/LzYwz63zuNzb/KphqYkQNgTWDloNRhuciu4NfGpKS3VV1qY8reMGVt9ZxJ/VxNJV4G+xDmsjNhvW9lKzPNlbj64Lg2Aq4mjz4WooUq7uXC+oi1kXctkcrpiyhXpYJoaWFrUQ8sgjFlZXYtNCPWpMLiigrXVcg6F1YFlCanNcVUteAkLJClvoWmSrRyJWzVvq/VAnSouFRJay1hGrWbFZzVpn64Ko42j+gRZmStAwLaJBJi05ZUOr5irWULUxVjOqgoyxmquYh5DBwSqHqGK1+Q+ZBlPdhlreU7HExontFkt2a9ZeeVjxU21OWXWoRWGwaWoOZnSgCSQWG1o21NpBy01EAWLLRcmQrg0A129XhYkrnV2JYWYrrkFMLcGhVZdqm9syjNaYAcM885M6uLrCnAIwTec2+FgBcgO6dTxc3ds8ZxkpS95KGrEQmvY9rhl+nQPmAbq+DhuNWoDDFAqYk9SmmecxSoQhGA0lgNib2xW5aCFoBUUJCq7L67BxZarsStcL9AqYXG2soM7mGsEL2W+i+YI7y9pFXcfnptuuUiTjQlgxDa1KWo0PJZBiGa4aF4pmuJpKC+qtE7tGXVrrYGBdBevFvljMURC1uUMju6B14rkeGPML1rL8vVpIgCoxmSBiTmYBbgmMpTk2gVUWc5uxAVIuvNKiJxoEtdBSKSCMv+gaTJeOrnx6wfC1bkE1MBIv0lsls4bRBSyrgGCOX4G+SYhLfGhRZfH8Cq5GbStrKH67XvJvjUbVEg2ilArdMuio6C1V2WupjK6w2npgc6nx5IpcWlmSttFaCUhVN7YJtYzNYtu2MLLSE2ghKGqfYAPQ4oUWVrHytvKeGBq2QhBDkRaIqv+pWPSz6Ni6v5qidWaTTSsj2MvPWq4kqitTNYs1t15hd0MLqqmVmMMSQFwlJ7UsttytzBRr/+KRRVHnyq9BUjpWu135rxCoRBAyPasCCC25qSEVzC1r3lKXgKgiQJtcLbusXV1bZP1Tx7JmLaYBKKTMQa4zk+uIq3HklkQAZF7cQF2Zlvs2YKsjUcmajb3Nnfqqzs3KX3wd3jpVLIrC1m8XjKzA2y5RpnGJEVXbUHMiIZNNVmniQn2qi2tBisWhxCjEKtosqLeat8Uvqy62KgRdjUGdH7T49yogWhXaepDM4NbIlQ5jXSjhs3qJ0SGs2toGcBmM8k2b4yswJ4VQ1e1MzahJZmOFZFdbRh6WtQImKWj7sqkIUCo0opEqaUgAqInHou0vzYNbBFvq5WRxowofC10x88gjmIFdCS2o2M2Nm7Y4ZC8XdDI3WoyLRoVKe5ZxXixnEWX9hUfLs6omKy9hpZS1tdS9NMCopS6TuqG2LuNIbYJY00qFBpsxGz0QWi5hUad8d1V+uvpZogpQqZlCaj6vVTazqNOcUNGyLbJ6bDVlDMYBmqFqKGtzZx227KqN2zWRrF7BiNhieEX1StVSMW59XWbLMkCroFNWJasJsgldYG2j1HSz5vYVUc10XBiQsbpyn1YppMpErYX1220FtvWjdWVpYR2xEtbFDNo4QvHbwiyoxv1WJLKaAtX0RSle8NdaKpVpVE9WmxbVtVZR22ZxsS21rGWJUs0fWta5ClMNVhcNikBeuYBdpUnaaAaqXksm9FTwqU0R1FVLiz4LYNhkEmgNayVOFwXn0Yyto1IHiSAWyqrs2fhcpf01MVATYZbbL0NojgiUMFSvqSYYGbBbo81NC+pKWcckWBQ2I1TLc/NmtRypuhKMXC4T2jK3om+QyZ8lyC4+bMyxxNI2DxfFsnkllcnd9GIb39UrwLL2pp03jxRbKENjMy0qVMPZtGuhpA3yOlhU77UigSWfba1/RFeKfZna29qYBRbr116vIRWVopqx7NLr+d8GywbVRqoJrxUYF05X8RpEpQyh2VNLine truncated
|
||||
"text/plain": [
|
||||
"<PIL.Image.Image image mode=RGB size=512x512>"
|
||||
]
|
||||
},
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"prompt = \"sks dog swimming in th pool with sunglasses\"\n",
|
||||
"image = pipe(prompt, num_inference_steps=50, guidance_scale=7.5, negative_prompt=negative_prompt).images[0]\n",
|
||||
"image"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 28,
|
||||
"id": "cca5106c",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "d9beb64979364f10b5cd9bba50d9caba",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
" 0%| | 0/50 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAIAAAB7GkOtAAEAAElEQVR4nFz9WZOtS3IdiC33+KY9ZOYZ7rm3CgRAgMWmOIBNmropqdU0mkzWDy39Oj3pQX9BZnqQyUyyplomqptNEc0mZoAAiCoUgKq605kycw/fEO568OWxs1Qgq+49J3Pv74vwcF9r+RDyf/g//p/ExKVAAXMIAIWbiwIuACCAAw6IwwXqDoELHCLmLqLuLhI/5xB1s/wtiIiZiyrcAQjEYAIRgbsDAhG4539DIADcIfEb8TdwtP+4x1PCAbjEzwkAQNqfQ/j/ISLuHk8jgKiamYq6m4i0P4WbCJzf7g6PhzSHSvwj4hX4n/gT8Nnz4fn9+WYCi9eW+F2Hx/Pk0znX58UieHy08/vyzzT+BKJcZwjc+RTxtvnYEJi78PvaYom3dXYXEW+LxYXLn4vdMX5X/nl8l4uo2W0dnEYCQNxMVN1jf8Xc4qnQ3lrE41PjGVTd3d1jhWMdnC/l+TAvttSN2wV3UX6Cmaq6GyAO0PY8fz3frH1Ue1carIibx6qoSntTM1NVLvNtl3MnY03N4kPiiwyAu3gz4dvR+KWvb7uTxs3FcUDjHwwQEanVpB3DZjYCLlE+G18pTkFaexzA+EnuOwAzUTUec0guUppls47YER673GwXiLdDZt6ORbxJfi8EcHERrbWGYQsAbfvb9rSto/OXVHjCHL/kTzz+V9zsxfblg7Ul4FrTsbS1obnGouWfupvGAkgcQ3V38XYo4nsN6RBEblud3xvboTziN0cQZ8TkdurzVdsS//KfxKHLbbW21/HDKmLxmbXGigsA1Wa36T3QLP/2H+5W+kl4Hu8wRevgxVXdDMa/peOGey6kQMwsTNHcRBUu5i5hmm5wfq8Dog5IRBC3MAXA3atpUYfBAIU73ztWKs9S7qe7O/eq2Q2XPHZZxBEb5m6uGsbsqgoBzF4eewGt0OMMhLUhQ5CLu9Go4msZiTwCgptLvFW6uvR0gIiGp3MXiLVjDsDg4uEgNY3czaFIGw2nEzYo7YXdDfwhmpm7aIEL3LztcCxCBKuIr+7+/3dI+BJm0Bak2uF7EXsA49mRtPBwsxbWCYOoSCyeIt1kOAi3DFk0RP4fj3I4IDdLi4db/q3F2XFz1/C+CGuxFkHzsNC2RdpmOeDh7iMu8JvziLdVjnMluMXW+J54fpF2iiKAwuFQCNSqNTdCk3F+Ig9tmDiXgKuNXBQCl3hmuFdGx9yLsEBaAsIiDbFy6SjDypu1RHSPDXsZ3HlOEUEwMVl6UNozQ7Eb0slC0s3/sqMD4NUiGolIrPnNdNMY+CfOT2p+0cxFxcUkgydjEwALj3F7Qs9I7JbxI534bVNyN2lL7dcByYNPM5B8qdiowLGMsrG2BlGHKwgBzTwiVqAsuAdok9uf8MmdXhMCMc+30wRADLFO44J6tdwCvwVmhnq52WS4pRbUW8ig4ySGzOCbK1LjRxVEDPFU9OvpxhKc8PTQR7vTFZhr5wZRSH5yPJ6q3tyHh49W5FO7u7nTcaG9BkDnkMghvKgCRt8EuKo6DPyzQNnaUGyLOYxzAjfTcHU8a+EXNF0NBNCi3hw64S1u0Cn9dZpX4Eie+TCYRA8RQ2HIz4nf0uZ3iBZFtYWI/EbCVjPXorAGN1z5Xi5AyYU1cxEVoY2bmwS7Sgdr5qqS2NDpEqXxDuSZERFoc3oeK+zNQeQLit0+OfcGNzzB5c4DGa4iIJ44XOmAVMTdLWKMuYio6I3WONxNRHncuc7h7kUFImpwEVh6ddU4t0LvHfui6ajNAeEbIZG1uagKAyAiskgzWIe5OaAEsdDwv+H8Y4slaYpKO9VhFaKiogKJ9UsK5O4BOFxE0inC3L3ejo+kCau3f5XYa0ZHua0tnYYGjYuj3BBh+Fdxd0sW3nYL5i/Qt0osPswdKs2B8hkNltvrjWg2n85VySfNCM6gxV+MGOy3d/QW5cDvQYDH3MFfOtDWNi4sImOYiriGX7ZqjGtpgo3rxL7Gv5oR+aVVcgnd7Rbd6ccQjEFUXChsxK8L6DqCFoiEebz0DG0DIt7lJ2sif1GE2xd4eowWrtrvGhG8ukLM0rDzRMdvGLdUAuM7HJZuSokXSUwF4houPl6U7s0bsxGhG3AzAFL0xaGSRvvi11TEYQpRKcXFtRGm9Ak3DhKnlJiOhD2OuJlHgJIEawm4GG+l4UMNGEh228yNAdAJnhyANubUrLn9oySmJ0lRvQkcGfz5w7g5dBEJkMifbMKGvNj7sKnYoXY4eQwiqMeSCwFdA8A8ThKHgFIYdRqaPV1ngzZwV024aCZKo/EXdqwCEQlf5nJjH2mERGdtV0CvIml47b0aA6WpiQZuyqAdS94WVQTpFpHoQ9J9e3qQONgCTU+WB1jyOCTdcHdRUiBi24YYwhsGeH8pOPGtuOfxppqrp0V/aa00EV5G3DjVgFBnEQSm8fxkYs8b3MjooZJMAiIhQtLZqSrEm22Y0W0nEuBJY6BScXDp/GYzMLI8umAassBuDi8+EIyIEh5KacnOfVVumHCzJBc7ZKi21u78XWknDIAY6EXQ8IHKjSPkeefOcym0OdZw0MoY7Gk26pJm0BRbgWg4TG4Wzd5J62nGGiorkvehrZg3aBX7ycjrduMFeTLS7eZKkK3BvRSC/fCzVumZGNvyUxwBX5JkQLku2g4fzNsbJ8B2z8OaX5rANI6JMKCkN3KDNH+VLjUUdZ5Z2vnt4CE52i3AwyxNXoNjSBOdEhwkBBRxN+Gq8k3zoFnnyWdVxarTsTmXMBbK0ysKbickkUXuZTMvN+H3hvBC9KjBIUBJLgwhQ2JE7BuQjZ91M9X0DgjTVB5gh5uJqCgIS7xJAHQ7YTC0IU3j9LQqAAJFiCc0CHhysVA8m/MiaktsFUdK1KuZ5OFShhlGac1HEqhqaFwRSsytKR4QJfaU/FpPeuQpRDbZN98hNjsJTBipaxE3VzTAy7/U0lQTT7SORGnuLpY2jRSh0h01LCBOHxoRnDHeYCGjeiZ9WtCNL6pWqc7RNxNWkwl5S8mIB1gMsVIg1HyJyq1CeQ4TTMa3tE13eFLg5gP5GDRWfjKxGRNaTWKiZxF6TspoAarEGYm5+g4VMTOE4dBvBmlAkQAAfEHL9E18lIf3UkDELWBBRsfbNsDNzYywiVJVkP2m0SSGzI174WNSwwPiQ0I+1kAlSQQBBKk1T/E9zcwRoiIc8OpagrcpgUTgcW+arfDpHUWDi8cDwqprScSQBugt12KEiQDBdp7iFmglfSyam6KjCJ/AUM34nfviHgCLGEa2as1NqItoZC5FVSxPdsOWJt5oHxNKyRIcKGQS8QiBWvL8EkvB5SbaSAIic1fhAiPipRH/O0ShBntx9vAiQdcYozu8KFMORTrzGpGYBuywlL6B4D2hBSUlSx8lgTnMRKTjX2XEByRdHr9TkkqnUwkGKkbH2eiSGtxh+c6EfkEdVJUPJw73oqVaVVG01FahpJCvjYDJydzBzwxHI7lpoXhrUtnU7dojh2eUPNdE8Y6Q5ZnG0AhBIopKKTxYo+Tyh42ZioZ8Ha4wXHPzRHGeGeNFBAohoAhf4qGDSSQFwH8FJAzXYUZNQFXc+YSq+YsWaCm0wAifGjJ6C1jEsGYAsV8kMCBusDTT0CVuOfDQORO3taNOshK+QEzMah5vek5lPpe+LLxhyz9HHjUk45BKSRYTy4Q/vBEyg2gBbp8Q2M/dVQGBhyiPm8ovnsxJHKHVQUwsxLF4woAPDAZIBHX739QW3KVkOYPQQ8FcVA0mgJgAsDybsXU1gUY8f1uaOMYBfxt9DgONgCwNLIehhq9wCKTCAKa1AkU6BM7KBYWQRihPBY0NgKNAIVwNIFfLrRRlws4r9x0oRdNVQdKIXvrrCGXiri0Bk+5FoS4N/0IAzZeM/2pe2PO8tjAmEXWIiIOP0lXxwwPk3pxZw5OedE9DdxIwFwpAVM1NAOZa0DCZw6VoMVQXZwh05mnAkgSIwEwihx8hX4BCABqHSNyqMvyIipjCzAJHNvDh7ioM8HBEArBovDhxEhIQgGEYImJWQ1ey0FFD4A37Vbp4zwx/K7VIx21hBAIxsxCFJXfWcSOvRYlmtCgc3W3HkFUZ4m5Uq51xPryguEcuS0RcHZASZ7UR28QHSSeZIoMJNwZw1RJpgBbtGVZi7fJcMNpLc/jeKGcppVrVzF+9ZA4CV1FzU1prOKaENxaQLOIu7UyS+gbtiSNXStdoo5tnbtTIolJcSwcYK6u3NwpPGk7OHCUYolKvBPFlhIuwUBUNN5ApTleRInkAcMuIeWb/gvkr/4RsLUCNOMytKyWWIXaswjJd4Mo0pohoZAwaSjCqwwiaIqKOSunRk0m6lfaLQR+pqCTqYRQMMsflTV7VlHvIL53zpsiktYZXEAjyVIhIlug4FeVkBh5hzGKt4/OtWoK7dNNxJNwMKE2DB0RL5C1VGEEJwN24+9JcrYvAttCauD9mNaKcJMwTKrbOzJhEihdtOfiSKRlo/AChC0SktgRdVnEIabyljQEODY6PROjpZRxRk6QaVhHIMfggf+CXuFSFqTYPSHONPYszFdxUKeg5ebXRleafKMkYGpxVACWzRMFUlF9CVy7I6GItDEhqQhGDIoCqQj3SIFlJJa0QQ7zIra7MItOLCtVaQ3VCiFIOCxBisUJtMRRFShxSuaFRB8NMKjJxboMKuIAvI2F+KhLrHZ7NmKH1jG0AWDYSuCM4hLmpllyiLhxdsudUNcLB0Zch1YX09WAxgpYSuy8k1uaRsmV4aMkAYs04YZ428+I8hzWaxS4byBr4kYIKY6R1UtubkhCohRGexzONkPYd8qgn80lglkap8Q8ScZT8VSWilEJUpYiqSqhMIqI8LF5CsiTlpT9N/AXnkybCDBsLw2zBIN40nSwoL8TZdPPG12IjuAaxuI3oB1duwo6GFti0U7hlvaw7XCjhKW6pCDNLine truncated
|
||||
"text/plain": [
|
||||
"<PIL.Image.Image image mode=RGB size=512x512>"
|
||||
]
|
||||
},
|
||||
"execution_count": 28,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"prompt = \"sks dog weaing red sweater\"\n",
|
||||
"image = pipe(prompt, num_inference_steps=50, guidance_scale=7.5, negative_prompt=negative_prompt).images[0]\n",
|
||||
"image"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "121b1da3",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "1f0ecb40",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
@@ -0,0 +1,993 @@
|
||||
import argparse
|
||||
import gc
|
||||
import hashlib
|
||||
import itertools
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import threading
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
import psutil
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import torch.utils.checkpoint
|
||||
from torch.utils.data import Dataset
|
||||
from dataclasses import asdict
|
||||
|
||||
import datasets
|
||||
import diffusers
|
||||
import transformers
|
||||
from accelerate import Accelerator
|
||||
from accelerate.logging import get_logger
|
||||
from accelerate.utils import set_seed
|
||||
from diffusers import AutoencoderKL, DDPMScheduler, DiffusionPipeline, UNet2DConditionModel
|
||||
from diffusers import DDPMScheduler, PNDMScheduler, StableDiffusionPipeline
|
||||
from diffusers.pipelines.stable_diffusion import StableDiffusionSafetyChecker
|
||||
from diffusers.optimization import get_scheduler
|
||||
from diffusers.utils import check_min_version
|
||||
from diffusers.utils.import_utils import is_xformers_available
|
||||
from huggingface_hub import HfFolder, Repository, whoami
|
||||
from PIL import Image
|
||||
from torchvision import transforms
|
||||
from tqdm.auto import tqdm
|
||||
from transformers import AutoTokenizer, PretrainedConfig, CLIPFeatureExtractor
|
||||
from pet import LoRAModel, LoRAConfig, get_pet_model_state_dict
|
||||
|
||||
# Will error if the minimal version of diffusers is not installed. Remove at your own risks.
|
||||
check_min_version("0.10.0.dev0")
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
UNET_TARGET_MODULES = ["to_q", "to_v", "query", "value"] # , "ff.net.0.proj"]
|
||||
TEXT_ENCODER_TARGET_MODULES = ["q_proj", "v_proj"]
|
||||
|
||||
|
||||
def import_model_class_from_model_name_or_path(pretrained_model_name_or_path: str, revision: str):
|
||||
text_encoder_config = PretrainedConfig.from_pretrained(
|
||||
pretrained_model_name_or_path,
|
||||
subfolder="text_encoder",
|
||||
revision=revision,
|
||||
)
|
||||
model_class = text_encoder_config.architectures[0]
|
||||
|
||||
if model_class == "CLIPTextModel":
|
||||
from transformers import CLIPTextModel
|
||||
|
||||
return CLIPTextModel
|
||||
elif model_class == "RobertaSeriesModelWithTransformation":
|
||||
from diffusers.pipelines.alt_diffusion.modeling_roberta_series import RobertaSeriesModelWithTransformation
|
||||
|
||||
return RobertaSeriesModelWithTransformation
|
||||
else:
|
||||
raise ValueError(f"{model_class} is not supported.")
|
||||
|
||||
|
||||
def parse_args(input_args=None):
|
||||
parser = argparse.ArgumentParser(description="Simple example of a training script.")
|
||||
parser.add_argument(
|
||||
"--pretrained_model_name_or_path",
|
||||
type=str,
|
||||
default=None,
|
||||
required=True,
|
||||
help="Path to pretrained model or model identifier from huggingface.co/models.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--revision",
|
||||
type=str,
|
||||
default=None,
|
||||
required=False,
|
||||
help="Revision of pretrained model identifier from huggingface.co/models.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tokenizer_name",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Pretrained tokenizer name or path if not the same as model_name",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--instance_data_dir",
|
||||
type=str,
|
||||
default=None,
|
||||
required=True,
|
||||
help="A folder containing the training data of instance images.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--class_data_dir",
|
||||
type=str,
|
||||
default=None,
|
||||
required=False,
|
||||
help="A folder containing the training data of class images.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--instance_prompt",
|
||||
type=str,
|
||||
default=None,
|
||||
required=True,
|
||||
help="The prompt with identifier specifying the instance",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--class_prompt",
|
||||
type=str,
|
||||
default=None,
|
||||
help="The prompt to specify images in the same class as provided instance images.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--with_prior_preservation",
|
||||
default=False,
|
||||
action="store_true",
|
||||
help="Flag to add prior preservation loss.",
|
||||
)
|
||||
parser.add_argument("--prior_loss_weight", type=float, default=1.0, help="The weight of prior preservation loss.")
|
||||
parser.add_argument(
|
||||
"--num_class_images",
|
||||
type=int,
|
||||
default=100,
|
||||
help=(
|
||||
"Minimal class images for prior preservation loss. If there are not enough images already present in"
|
||||
" class_data_dir, additional images will be sampled with class_prompt."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output_dir",
|
||||
type=str,
|
||||
default="text-inversion-model",
|
||||
help="The output directory where the model predictions and checkpoints will be written.",
|
||||
)
|
||||
parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.")
|
||||
parser.add_argument(
|
||||
"--resolution",
|
||||
type=int,
|
||||
default=512,
|
||||
help=(
|
||||
"The resolution for input images, all the images in the train/validation dataset will be resized to this"
|
||||
" resolution"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--center_crop", action="store_true", help="Whether to center crop images before resizing to resolution"
|
||||
)
|
||||
parser.add_argument("--train_text_encoder", action="store_true", help="Whether to train the text encoder")
|
||||
|
||||
# lora args
|
||||
parser.add_argument("--use_lora", action="store_true", help="Whether to use LoRA for parameter efficient tuning")
|
||||
parser.add_argument("--lora_r", type=int, default=8, help="LoRA rank, only used if use_lora is True")
|
||||
parser.add_argument("--lora_alpha", type=int, default=32, help="LoRA alpha, only used if use_lora is True")
|
||||
parser.add_argument("--lora_dropout", type=float, default=0.0, help="LoRA dropout, only used if use_lora is True")
|
||||
parser.add_argument(
|
||||
"--lora_text_encoder_r",
|
||||
type=int,
|
||||
default=8,
|
||||
help="LoRA rank for text encoder, only used if `use_lora` and `train_text_encoder` are True",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lora_text_encoder_alpha",
|
||||
type=int,
|
||||
default=32,
|
||||
help="LoRA alpha for text encoder, only used if `use_lora` and `train_text_encoder` are True",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lora_text_encoder_dropout",
|
||||
type=float,
|
||||
default=0.0,
|
||||
help="LoRA dropout for text encoder, only used if `use_lora` and `train_text_encoder` are True",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--train_batch_size", type=int, default=4, help="Batch size (per device) for the training dataloader."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--sample_batch_size", type=int, default=4, help="Batch size (per device) for sampling images."
|
||||
)
|
||||
parser.add_argument("--num_train_epochs", type=int, default=1)
|
||||
parser.add_argument(
|
||||
"--max_train_steps",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Total number of training steps to perform. If provided, overrides num_train_epochs.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--checkpointing_steps",
|
||||
type=int,
|
||||
default=500,
|
||||
help=(
|
||||
"Save a checkpoint of the training state every X updates. These checkpoints can be used both as final"
|
||||
" checkpoints in case they are better than the last checkpoint, and are also suitable for resuming"
|
||||
" training using `--resume_from_checkpoint`."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--resume_from_checkpoint",
|
||||
type=str,
|
||||
default=None,
|
||||
help=(
|
||||
"Whether training should be resumed from a previous checkpoint. Use a path saved by"
|
||||
' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.'
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--gradient_accumulation_steps",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of updates steps to accumulate before performing a backward/update pass.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--gradient_checkpointing",
|
||||
action="store_true",
|
||||
help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--learning_rate",
|
||||
type=float,
|
||||
default=5e-6,
|
||||
help="Initial learning rate (after the potential warmup period) to use.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--scale_lr",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lr_scheduler",
|
||||
type=str,
|
||||
default="constant",
|
||||
help=(
|
||||
'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",'
|
||||
' "constant", "constant_with_warmup"]'
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lr_num_cycles",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of hard resets of the lr in cosine_with_restarts scheduler.",
|
||||
)
|
||||
parser.add_argument("--lr_power", type=float, default=1.0, help="Power factor of the polynomial scheduler.")
|
||||
parser.add_argument(
|
||||
"--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes."
|
||||
)
|
||||
parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.")
|
||||
parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.")
|
||||
parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.")
|
||||
parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer")
|
||||
parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.")
|
||||
parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.")
|
||||
parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.")
|
||||
parser.add_argument(
|
||||
"--hub_model_id",
|
||||
type=str,
|
||||
default=None,
|
||||
help="The name of the repository to keep in sync with the local `output_dir`.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--logging_dir",
|
||||
type=str,
|
||||
default="logs",
|
||||
help=(
|
||||
"[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to"
|
||||
" *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--allow_tf32",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see"
|
||||
" https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--report_to",
|
||||
type=str,
|
||||
default="tensorboard",
|
||||
help=(
|
||||
'The integration to report the results and logs to. Supported platforms are `"tensorboard"`'
|
||||
' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.'
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mixed_precision",
|
||||
type=str,
|
||||
default=None,
|
||||
choices=["no", "fp16", "bf16"],
|
||||
help=(
|
||||
"Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >="
|
||||
" 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the"
|
||||
" flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prior_generation_precision",
|
||||
type=str,
|
||||
default=None,
|
||||
choices=["no", "fp32", "fp16", "bf16"],
|
||||
help=(
|
||||
"Choose prior generation precision between fp32, fp16 and bf16 (bfloat16). Bf16 requires PyTorch >="
|
||||
" 1.10.and an Nvidia Ampere GPU. Default to fp16 if a GPU is available else fp32."
|
||||
),
|
||||
)
|
||||
parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank")
|
||||
parser.add_argument(
|
||||
"--enable_xformers_memory_efficient_attention", action="store_true", help="Whether or not to use xformers."
|
||||
)
|
||||
|
||||
if input_args is not None:
|
||||
args = parser.parse_args(input_args)
|
||||
else:
|
||||
args = parser.parse_args()
|
||||
|
||||
env_local_rank = int(os.environ.get("LOCAL_RANK", -1))
|
||||
if env_local_rank != -1 and env_local_rank != args.local_rank:
|
||||
args.local_rank = env_local_rank
|
||||
|
||||
if args.with_prior_preservation:
|
||||
if args.class_data_dir is None:
|
||||
raise ValueError("You must specify a data directory for class images.")
|
||||
if args.class_prompt is None:
|
||||
raise ValueError("You must specify prompt for class images.")
|
||||
else:
|
||||
# logger is not available yet
|
||||
if args.class_data_dir is not None:
|
||||
warnings.warn("You need not use --class_data_dir without --with_prior_preservation.")
|
||||
if args.class_prompt is not None:
|
||||
warnings.warn("You need not use --class_prompt without --with_prior_preservation.")
|
||||
|
||||
return args
|
||||
|
||||
|
||||
# Converting Bytes to Megabytes
|
||||
def b2mb(x):
|
||||
return int(x / 2**20)
|
||||
|
||||
|
||||
# This context manager is used to track the peak memory usage of the process
|
||||
class TorchTracemalloc:
|
||||
def __enter__(self):
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
torch.cuda.reset_max_memory_allocated() # reset the peak gauge to zero
|
||||
self.begin = torch.cuda.memory_allocated()
|
||||
self.process = psutil.Process()
|
||||
|
||||
self.cpu_begin = self.cpu_mem_used()
|
||||
self.peak_monitoring = True
|
||||
peak_monitor_thread = threading.Thread(target=self.peak_monitor_func)
|
||||
peak_monitor_thread.daemon = True
|
||||
peak_monitor_thread.start()
|
||||
return self
|
||||
|
||||
def cpu_mem_used(self):
|
||||
"""get resident set size memory for the current process"""
|
||||
return self.process.memory_info().rss
|
||||
|
||||
def peak_monitor_func(self):
|
||||
self.cpu_peak = -1
|
||||
|
||||
while True:
|
||||
self.cpu_peak = max(self.cpu_mem_used(), self.cpu_peak)
|
||||
|
||||
# can't sleep or will not catch the peak right (this comment is here on purpose)
|
||||
# time.sleep(0.001) # 1msec
|
||||
|
||||
if not self.peak_monitoring:
|
||||
break
|
||||
|
||||
def __exit__(self, *exc):
|
||||
self.peak_monitoring = False
|
||||
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
self.end = torch.cuda.memory_allocated()
|
||||
self.peak = torch.cuda.max_memory_allocated()
|
||||
self.used = b2mb(self.end - self.begin)
|
||||
self.peaked = b2mb(self.peak - self.begin)
|
||||
|
||||
self.cpu_end = self.cpu_mem_used()
|
||||
self.cpu_used = b2mb(self.cpu_end - self.cpu_begin)
|
||||
self.cpu_peaked = b2mb(self.cpu_peak - self.cpu_begin)
|
||||
# print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")
|
||||
|
||||
|
||||
def print_trainable_parameters(model):
|
||||
"""
|
||||
Prints the number of trainable parameters in the model.
|
||||
"""
|
||||
trainable_params = 0
|
||||
all_param = 0
|
||||
for _, param in model.named_parameters():
|
||||
all_param += param.numel()
|
||||
if param.requires_grad:
|
||||
trainable_params += param.numel()
|
||||
print(
|
||||
f"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}"
|
||||
)
|
||||
|
||||
|
||||
class DreamBoothDataset(Dataset):
|
||||
"""
|
||||
A dataset to prepare the instance and class images with the prompts for fine-tuning the model.
|
||||
It pre-processes the images and the tokenizes prompts.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
instance_data_root,
|
||||
instance_prompt,
|
||||
tokenizer,
|
||||
class_data_root=None,
|
||||
class_prompt=None,
|
||||
size=512,
|
||||
center_crop=False,
|
||||
):
|
||||
self.size = size
|
||||
self.center_crop = center_crop
|
||||
self.tokenizer = tokenizer
|
||||
|
||||
self.instance_data_root = Path(instance_data_root)
|
||||
if not self.instance_data_root.exists():
|
||||
raise ValueError("Instance images root doesn't exists.")
|
||||
|
||||
self.instance_images_path = list(Path(instance_data_root).iterdir())
|
||||
self.num_instance_images = len(self.instance_images_path)
|
||||
self.instance_prompt = instance_prompt
|
||||
self._length = self.num_instance_images
|
||||
|
||||
if class_data_root is not None:
|
||||
self.class_data_root = Path(class_data_root)
|
||||
self.class_data_root.mkdir(parents=True, exist_ok=True)
|
||||
self.class_images_path = list(self.class_data_root.iterdir())
|
||||
self.num_class_images = len(self.class_images_path)
|
||||
self._length = max(self.num_class_images, self.num_instance_images)
|
||||
self.class_prompt = class_prompt
|
||||
else:
|
||||
self.class_data_root = None
|
||||
|
||||
self.image_transforms = transforms.Compose(
|
||||
[
|
||||
transforms.Resize(size, interpolation=transforms.InterpolationMode.BILINEAR),
|
||||
transforms.CenterCrop(size) if center_crop else transforms.RandomCrop(size),
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize([0.5], [0.5]),
|
||||
]
|
||||
)
|
||||
|
||||
def __len__(self):
|
||||
return self._length
|
||||
|
||||
def __getitem__(self, index):
|
||||
example = {}
|
||||
instance_image = Image.open(self.instance_images_path[index % self.num_instance_images])
|
||||
if not instance_image.mode == "RGB":
|
||||
instance_image = instance_image.convert("RGB")
|
||||
example["instance_images"] = self.image_transforms(instance_image)
|
||||
example["instance_prompt_ids"] = self.tokenizer(
|
||||
self.instance_prompt,
|
||||
truncation=True,
|
||||
padding="max_length",
|
||||
max_length=self.tokenizer.model_max_length,
|
||||
return_tensors="pt",
|
||||
).input_ids
|
||||
|
||||
if self.class_data_root:
|
||||
class_image = Image.open(self.class_images_path[index % self.num_class_images])
|
||||
if not class_image.mode == "RGB":
|
||||
class_image = class_image.convert("RGB")
|
||||
example["class_images"] = self.image_transforms(class_image)
|
||||
example["class_prompt_ids"] = self.tokenizer(
|
||||
self.class_prompt,
|
||||
truncation=True,
|
||||
padding="max_length",
|
||||
max_length=self.tokenizer.model_max_length,
|
||||
return_tensors="pt",
|
||||
).input_ids
|
||||
|
||||
return example
|
||||
|
||||
|
||||
def collate_fn(examples, with_prior_preservation=False):
|
||||
input_ids = [example["instance_prompt_ids"] for example in examples]
|
||||
pixel_values = [example["instance_images"] for example in examples]
|
||||
|
||||
# Concat class and instance examples for prior preservation.
|
||||
# We do this to avoid doing two forward passes.
|
||||
if with_prior_preservation:
|
||||
input_ids += [example["class_prompt_ids"] for example in examples]
|
||||
pixel_values += [example["class_images"] for example in examples]
|
||||
|
||||
pixel_values = torch.stack(pixel_values)
|
||||
pixel_values = pixel_values.to(memory_format=torch.contiguous_format).float()
|
||||
|
||||
input_ids = torch.cat(input_ids, dim=0)
|
||||
|
||||
batch = {
|
||||
"input_ids": input_ids,
|
||||
"pixel_values": pixel_values,
|
||||
}
|
||||
return batch
|
||||
|
||||
|
||||
class PromptDataset(Dataset):
|
||||
"A simple dataset to prepare the prompts to generate class images on multiple GPUs."
|
||||
|
||||
def __init__(self, prompt, num_samples):
|
||||
self.prompt = prompt
|
||||
self.num_samples = num_samples
|
||||
|
||||
def __len__(self):
|
||||
return self.num_samples
|
||||
|
||||
def __getitem__(self, index):
|
||||
example = {}
|
||||
example["prompt"] = self.prompt
|
||||
example["index"] = index
|
||||
return example
|
||||
|
||||
|
||||
def get_full_repo_name(model_id: str, organization: Optional[str] = None, token: Optional[str] = None):
|
||||
if token is None:
|
||||
token = HfFolder.get_token()
|
||||
if organization is None:
|
||||
username = whoami(token)["name"]
|
||||
return f"{username}/{model_id}"
|
||||
else:
|
||||
return f"{organization}/{model_id}"
|
||||
|
||||
|
||||
def main(args):
|
||||
logging_dir = Path(args.output_dir, args.logging_dir)
|
||||
|
||||
accelerator = Accelerator(
|
||||
gradient_accumulation_steps=args.gradient_accumulation_steps,
|
||||
mixed_precision=args.mixed_precision,
|
||||
log_with=args.report_to,
|
||||
logging_dir=logging_dir,
|
||||
)
|
||||
|
||||
# Currently, it's not possible to do gradient accumulation when training two models with accelerate.accumulate
|
||||
# This will be enabled soon in accelerate. For now, we don't allow gradient accumulation when training two models.
|
||||
# TODO (patil-suraj): Remove this check when gradient accumulation with two models is enabled in accelerate.
|
||||
if args.train_text_encoder and args.gradient_accumulation_steps > 1 and accelerator.num_processes > 1:
|
||||
raise ValueError(
|
||||
"Gradient accumulation is not supported when training the text encoder in distributed training. "
|
||||
"Please set gradient_accumulation_steps to 1. This feature will be supported in the future."
|
||||
)
|
||||
|
||||
# Make one log on every process with the configuration for debugging.
|
||||
logging.basicConfig(
|
||||
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
|
||||
datefmt="%m/%d/%Y %H:%M:%S",
|
||||
level=logging.INFO,
|
||||
)
|
||||
logger.info(accelerator.state, main_process_only=False)
|
||||
if accelerator.is_local_main_process:
|
||||
datasets.utils.logging.set_verbosity_warning()
|
||||
transformers.utils.logging.set_verbosity_warning()
|
||||
diffusers.utils.logging.set_verbosity_info()
|
||||
else:
|
||||
datasets.utils.logging.set_verbosity_error()
|
||||
transformers.utils.logging.set_verbosity_error()
|
||||
diffusers.utils.logging.set_verbosity_error()
|
||||
|
||||
# If passed along, set the training seed now.
|
||||
if args.seed is not None:
|
||||
set_seed(args.seed)
|
||||
|
||||
# Generate class images if prior preservation is enabled.
|
||||
if args.with_prior_preservation:
|
||||
class_images_dir = Path(args.class_data_dir)
|
||||
if not class_images_dir.exists():
|
||||
class_images_dir.mkdir(parents=True)
|
||||
cur_class_images = len(list(class_images_dir.iterdir()))
|
||||
|
||||
if cur_class_images < args.num_class_images:
|
||||
torch_dtype = torch.float16 if accelerator.device.type == "cuda" else torch.float32
|
||||
if args.prior_generation_precision == "fp32":
|
||||
torch_dtype = torch.float32
|
||||
elif args.prior_generation_precision == "fp16":
|
||||
torch_dtype = torch.float16
|
||||
elif args.prior_generation_precision == "bf16":
|
||||
torch_dtype = torch.bfloat16
|
||||
pipeline = DiffusionPipeline.from_pretrained(
|
||||
args.pretrained_model_name_or_path,
|
||||
torch_dtype=torch_dtype,
|
||||
safety_checker=None,
|
||||
revision=args.revision,
|
||||
)
|
||||
pipeline.set_progress_bar_config(disable=True)
|
||||
|
||||
num_new_images = args.num_class_images - cur_class_images
|
||||
logger.info(f"Number of class images to sample: {num_new_images}.")
|
||||
|
||||
sample_dataset = PromptDataset(args.class_prompt, num_new_images)
|
||||
sample_dataloader = torch.utils.data.DataLoader(sample_dataset, batch_size=args.sample_batch_size)
|
||||
|
||||
sample_dataloader = accelerator.prepare(sample_dataloader)
|
||||
pipeline.to(accelerator.device)
|
||||
|
||||
for example in tqdm(
|
||||
sample_dataloader, desc="Generating class images", disable=not accelerator.is_local_main_process
|
||||
):
|
||||
images = pipeline(example["prompt"]).images
|
||||
|
||||
for i, image in enumerate(images):
|
||||
hash_image = hashlib.sha1(image.tobytes()).hexdigest()
|
||||
image_filename = class_images_dir / f"{example['index'][i] + cur_class_images}-{hash_image}.jpg"
|
||||
image.save(image_filename)
|
||||
|
||||
del pipeline
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
# Handle the repository creation
|
||||
if accelerator.is_main_process:
|
||||
if args.push_to_hub:
|
||||
if args.hub_model_id is None:
|
||||
repo_name = get_full_repo_name(Path(args.output_dir).name, token=args.hub_token)
|
||||
else:
|
||||
repo_name = args.hub_model_id
|
||||
repo = Repository(args.output_dir, clone_from=repo_name)
|
||||
|
||||
with open(os.path.join(args.output_dir, ".gitignore"), "w+") as gitignore:
|
||||
if "step_*" not in gitignore:
|
||||
gitignore.write("step_*\n")
|
||||
if "epoch_*" not in gitignore:
|
||||
gitignore.write("epoch_*\n")
|
||||
elif args.output_dir is not None:
|
||||
os.makedirs(args.output_dir, exist_ok=True)
|
||||
|
||||
# Load the tokenizer
|
||||
if args.tokenizer_name:
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name, revision=args.revision, use_fast=False)
|
||||
elif args.pretrained_model_name_or_path:
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
args.pretrained_model_name_or_path,
|
||||
subfolder="tokenizer",
|
||||
revision=args.revision,
|
||||
use_fast=False,
|
||||
)
|
||||
|
||||
# import correct text encoder class
|
||||
text_encoder_cls = import_model_class_from_model_name_or_path(args.pretrained_model_name_or_path, args.revision)
|
||||
|
||||
# Load scheduler and models
|
||||
noise_scheduler = DDPMScheduler(
|
||||
beta_start=0.00085,
|
||||
beta_end=0.012,
|
||||
beta_schedule="scaled_linear",
|
||||
num_train_timesteps=1000,
|
||||
) # DDPMScheduler.from_pretrained(args.pretrained_model_name_or_path, subfolder="scheduler")
|
||||
text_encoder = text_encoder_cls.from_pretrained(
|
||||
args.pretrained_model_name_or_path, subfolder="text_encoder", revision=args.revision
|
||||
)
|
||||
vae = AutoencoderKL.from_pretrained(args.pretrained_model_name_or_path, subfolder="vae", revision=args.revision)
|
||||
unet = UNet2DConditionModel.from_pretrained(
|
||||
args.pretrained_model_name_or_path, subfolder="unet", revision=args.revision
|
||||
)
|
||||
feature_extractor = CLIPFeatureExtractor.from_pretrained("openai/clip-vit-base-patch32")
|
||||
|
||||
if args.use_lora:
|
||||
config = LoRAConfig(
|
||||
r=args.lora_r,
|
||||
lora_alpha=args.lora_alpha,
|
||||
target_modules=UNET_TARGET_MODULES,
|
||||
lora_dropout=args.lora_dropout,
|
||||
)
|
||||
unet = LoRAModel(config, unet)
|
||||
print_trainable_parameters(unet)
|
||||
print(unet)
|
||||
|
||||
vae.requires_grad_(False)
|
||||
if not args.train_text_encoder:
|
||||
text_encoder.requires_grad_(False)
|
||||
elif args.train_text_encoder and args.use_lora:
|
||||
config = LoRAConfig(
|
||||
r=args.lora_text_encoder_r,
|
||||
lora_alpha=args.lora_text_encoder_alpha,
|
||||
target_modules=TEXT_ENCODER_TARGET_MODULES,
|
||||
lora_dropout=args.lora_text_encoder_dropout,
|
||||
)
|
||||
text_encoder = LoRAModel(config, text_encoder)
|
||||
print_trainable_parameters(text_encoder)
|
||||
print(text_encoder)
|
||||
|
||||
if args.enable_xformers_memory_efficient_attention:
|
||||
if is_xformers_available():
|
||||
unet.enable_xformers_memory_efficient_attention()
|
||||
else:
|
||||
raise ValueError("xformers is not available. Make sure it is installed correctly")
|
||||
|
||||
if args.gradient_checkpointing:
|
||||
unet.enable_gradient_checkpointing()
|
||||
if args.train_text_encoder:
|
||||
text_encoder.gradient_checkpointing_enable()
|
||||
|
||||
# Enable TF32 for faster training on Ampere GPUs,
|
||||
# cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices
|
||||
if args.allow_tf32:
|
||||
torch.backends.cuda.matmul.allow_tf32 = True
|
||||
|
||||
if args.scale_lr:
|
||||
args.learning_rate = (
|
||||
args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes
|
||||
)
|
||||
|
||||
# Use 8-bit Adam for lower memory usage or to fine-tune the model in 16GB GPUs
|
||||
if args.use_8bit_adam:
|
||||
try:
|
||||
import bitsandbytes as bnb
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"To use 8-bit Adam, please install the bitsandbytes library: `pip install bitsandbytes`."
|
||||
)
|
||||
|
||||
optimizer_class = bnb.optim.AdamW8bit
|
||||
else:
|
||||
optimizer_class = torch.optim.AdamW
|
||||
|
||||
# Optimizer creation
|
||||
params_to_optimize = (
|
||||
itertools.chain(unet.parameters(), text_encoder.parameters()) if args.train_text_encoder else unet.parameters()
|
||||
)
|
||||
optimizer = optimizer_class(
|
||||
params_to_optimize,
|
||||
lr=args.learning_rate,
|
||||
betas=(args.adam_beta1, args.adam_beta2),
|
||||
weight_decay=args.adam_weight_decay,
|
||||
eps=args.adam_epsilon,
|
||||
)
|
||||
|
||||
# Dataset and DataLoaders creation:
|
||||
train_dataset = DreamBoothDataset(
|
||||
instance_data_root=args.instance_data_dir,
|
||||
instance_prompt=args.instance_prompt,
|
||||
class_data_root=args.class_data_dir if args.with_prior_preservation else None,
|
||||
class_prompt=args.class_prompt,
|
||||
tokenizer=tokenizer,
|
||||
size=args.resolution,
|
||||
center_crop=args.center_crop,
|
||||
)
|
||||
|
||||
train_dataloader = torch.utils.data.DataLoader(
|
||||
train_dataset,
|
||||
batch_size=args.train_batch_size,
|
||||
shuffle=True,
|
||||
collate_fn=lambda examples: collate_fn(examples, args.with_prior_preservation),
|
||||
num_workers=1,
|
||||
)
|
||||
|
||||
# Scheduler and math around the number of training steps.
|
||||
overrode_max_train_steps = False
|
||||
num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)
|
||||
if args.max_train_steps is None:
|
||||
args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch
|
||||
overrode_max_train_steps = True
|
||||
|
||||
lr_scheduler = get_scheduler(
|
||||
args.lr_scheduler,
|
||||
optimizer=optimizer,
|
||||
num_warmup_steps=args.lr_warmup_steps * args.gradient_accumulation_steps,
|
||||
num_training_steps=args.max_train_steps * args.gradient_accumulation_steps,
|
||||
num_cycles=args.lr_num_cycles,
|
||||
power=args.lr_power,
|
||||
)
|
||||
|
||||
# Prepare everything with our `accelerator`.
|
||||
if args.train_text_encoder:
|
||||
unet, text_encoder, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(
|
||||
unet, text_encoder, optimizer, train_dataloader, lr_scheduler
|
||||
)
|
||||
else:
|
||||
unet, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(
|
||||
unet, optimizer, train_dataloader, lr_scheduler
|
||||
)
|
||||
|
||||
# For mixed precision training we cast the text_encoder and vae weights to half-precision
|
||||
# as these models are only used for inference, keeping weights in full precision is not required.
|
||||
weight_dtype = torch.float32
|
||||
if accelerator.mixed_precision == "fp16":
|
||||
weight_dtype = torch.float16
|
||||
elif accelerator.mixed_precision == "bf16":
|
||||
weight_dtype = torch.bfloat16
|
||||
|
||||
# Move vae and text_encoder to device and cast to weight_dtype
|
||||
vae.to(accelerator.device, dtype=weight_dtype)
|
||||
if not args.train_text_encoder:
|
||||
text_encoder.to(accelerator.device, dtype=weight_dtype)
|
||||
|
||||
# We need to recalculate our total training steps as the size of the training dataloader may have changed.
|
||||
num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)
|
||||
if overrode_max_train_steps:
|
||||
args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch
|
||||
# Afterwards we recalculate our number of training epochs
|
||||
args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)
|
||||
|
||||
# We need to initialize the trackers we use, and also store our configuration.
|
||||
# The trackers initializes automatically on the main process.
|
||||
if accelerator.is_main_process:
|
||||
accelerator.init_trackers("dreambooth", config=vars(args))
|
||||
|
||||
# Train!
|
||||
total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps
|
||||
|
||||
logger.info("***** Running training *****")
|
||||
logger.info(f" Num examples = {len(train_dataset)}")
|
||||
logger.info(f" Num batches each epoch = {len(train_dataloader)}")
|
||||
logger.info(f" Num Epochs = {args.num_train_epochs}")
|
||||
logger.info(f" Instantaneous batch size per device = {args.train_batch_size}")
|
||||
logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}")
|
||||
logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}")
|
||||
logger.info(f" Total optimization steps = {args.max_train_steps}")
|
||||
global_step = 0
|
||||
first_epoch = 0
|
||||
|
||||
# Potentially load in the weights and states from a previous save
|
||||
if args.resume_from_checkpoint:
|
||||
if args.resume_from_checkpoint != "latest":
|
||||
path = os.path.basename(args.resume_from_checkpoint)
|
||||
else:
|
||||
# Get the mos recent checkpoint
|
||||
dirs = os.listdir(args.output_dir)
|
||||
dirs = [d for d in dirs if d.startswith("checkpoint")]
|
||||
dirs = sorted(dirs, key=lambda x: int(x.split("-")[1]))
|
||||
path = dirs[-1]
|
||||
accelerator.print(f"Resuming from checkpoint {path}")
|
||||
accelerator.load_state(os.path.join(args.output_dir, path))
|
||||
global_step = int(path.split("-")[1])
|
||||
|
||||
resume_global_step = global_step * args.gradient_accumulation_steps
|
||||
first_epoch = resume_global_step // num_update_steps_per_epoch
|
||||
resume_step = resume_global_step % num_update_steps_per_epoch
|
||||
|
||||
# Only show the progress bar once on each machine.
|
||||
progress_bar = tqdm(range(global_step, args.max_train_steps), disable=not accelerator.is_local_main_process)
|
||||
progress_bar.set_description("Steps")
|
||||
|
||||
for epoch in range(first_epoch, args.num_train_epochs):
|
||||
unet.train()
|
||||
if args.train_text_encoder:
|
||||
text_encoder.train()
|
||||
with TorchTracemalloc() as tracemalloc:
|
||||
for step, batch in enumerate(train_dataloader):
|
||||
# Skip steps until we reach the resumed step
|
||||
if args.resume_from_checkpoint and epoch == first_epoch and step < resume_step:
|
||||
if step % args.gradient_accumulation_steps == 0:
|
||||
progress_bar.update(1)
|
||||
continue
|
||||
|
||||
with accelerator.accumulate(unet):
|
||||
# Convert images to latent space
|
||||
latents = vae.encode(batch["pixel_values"].to(dtype=weight_dtype)).latent_dist.sample()
|
||||
latents = latents * 0.18215
|
||||
|
||||
# Sample noise that we'll add to the latents
|
||||
noise = torch.randn_like(latents)
|
||||
bsz = latents.shape[0]
|
||||
# Sample a random timestep for each image
|
||||
timesteps = torch.randint(
|
||||
0, noise_scheduler.config.num_train_timesteps, (bsz,), device=latents.device
|
||||
)
|
||||
timesteps = timesteps.long()
|
||||
|
||||
# Add noise to the latents according to the noise magnitude at each timestep
|
||||
# (this is the forward diffusion process)
|
||||
noisy_latents = noise_scheduler.add_noise(latents, noise, timesteps)
|
||||
|
||||
# Get the text embedding for conditioning
|
||||
encoder_hidden_states = text_encoder(batch["input_ids"])[0]
|
||||
|
||||
# Predict the noise residual
|
||||
model_pred = unet(noisy_latents, timesteps, encoder_hidden_states).sample
|
||||
|
||||
# Get the target for loss depending on the prediction type
|
||||
if noise_scheduler.config.prediction_type == "epsilon":
|
||||
target = noise
|
||||
elif noise_scheduler.config.prediction_type == "v_prediction":
|
||||
target = noise_scheduler.get_velocity(latents, noise, timesteps)
|
||||
else:
|
||||
raise ValueError(f"Unknown prediction type {noise_scheduler.config.prediction_type}")
|
||||
|
||||
if args.with_prior_preservation:
|
||||
# Chunk the noise and model_pred into two parts and compute the loss on each part separately.
|
||||
model_pred, model_pred_prior = torch.chunk(model_pred, 2, dim=0)
|
||||
target, target_prior = torch.chunk(target, 2, dim=0)
|
||||
|
||||
# Compute instance loss
|
||||
loss = F.mse_loss(model_pred.float(), target.float(), reduction="mean")
|
||||
|
||||
# Compute prior loss
|
||||
prior_loss = F.mse_loss(model_pred_prior.float(), target_prior.float(), reduction="mean")
|
||||
|
||||
# Add the prior loss to the instance loss.
|
||||
loss = loss + args.prior_loss_weight * prior_loss
|
||||
else:
|
||||
loss = F.mse_loss(model_pred.float(), target.float(), reduction="mean")
|
||||
|
||||
accelerator.backward(loss)
|
||||
if accelerator.sync_gradients:
|
||||
params_to_clip = (
|
||||
itertools.chain(unet.parameters(), text_encoder.parameters())
|
||||
if args.train_text_encoder
|
||||
else unet.parameters()
|
||||
)
|
||||
accelerator.clip_grad_norm_(params_to_clip, args.max_grad_norm)
|
||||
optimizer.step()
|
||||
lr_scheduler.step()
|
||||
optimizer.zero_grad()
|
||||
|
||||
# Checks if the accelerator has performed an optimization step behind the scenes
|
||||
if accelerator.sync_gradients:
|
||||
progress_bar.update(1)
|
||||
global_step += 1
|
||||
|
||||
# if global_step % args.checkpointing_steps == 0:
|
||||
# if accelerator.is_main_process:
|
||||
# save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}")
|
||||
# accelerator.save_state(save_path)
|
||||
# logger.info(f"Saved state to {save_path}")
|
||||
|
||||
logs = {"loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]}
|
||||
progress_bar.set_postfix(**logs)
|
||||
accelerator.log(logs, step=global_step)
|
||||
|
||||
if global_step >= args.max_train_steps:
|
||||
break
|
||||
# Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
|
||||
accelerator.print("GPU Memory before entering the train : {}".format(b2mb(tracemalloc.begin)))
|
||||
accelerator.print("GPU Memory consumed at the end of the train (end-begin): {}".format(tracemalloc.used))
|
||||
accelerator.print("GPU Peak Memory consumed during the train (max-begin): {}".format(tracemalloc.peaked))
|
||||
accelerator.print(
|
||||
"GPU Total Peak Memory consumed during the train (max): {}".format(
|
||||
tracemalloc.peaked + b2mb(tracemalloc.begin)
|
||||
)
|
||||
)
|
||||
|
||||
accelerator.print("CPU Memory before entering the train : {}".format(b2mb(tracemalloc.cpu_begin)))
|
||||
accelerator.print("CPU Memory consumed at the end of the train (end-begin): {}".format(tracemalloc.cpu_used))
|
||||
accelerator.print("CPU Peak Memory consumed during the train (max-begin): {}".format(tracemalloc.cpu_peaked))
|
||||
accelerator.print(
|
||||
"CPU Total Peak Memory consumed during the train (max): {}".format(
|
||||
tracemalloc.cpu_peaked + b2mb(tracemalloc.cpu_begin)
|
||||
)
|
||||
)
|
||||
|
||||
# Create the pipeline using using the trained modules and save it.
|
||||
accelerator.wait_for_everyone()
|
||||
if accelerator.is_main_process:
|
||||
if args.use_lora:
|
||||
lora_config = {}
|
||||
state_dict = get_pet_model_state_dict(unet, state_dict=accelerator.get_state_dict(unet))
|
||||
lora_config["pet_config"] = unet.get_pet_config_as_dict(inference=True)
|
||||
if args.train_text_encoder:
|
||||
text_encoder_state_dict = get_pet_model_state_dict(
|
||||
text_encoder, state_dict=accelerator.get_state_dict(text_encoder)
|
||||
)
|
||||
text_encoder_state_dict = {f"text_encoder_{k}": v for k, v in text_encoder_state_dict.items()}
|
||||
state_dict.update(text_encoder_state_dict)
|
||||
lora_config["text_encoder_pet_config"] = text_encoder.get_pet_config_as_dict(inference=True)
|
||||
|
||||
accelerator.print(state_dict)
|
||||
accelerator.save(state_dict, os.path.join(args.output_dir, f"{args.instance_prompt}_lora.pt"))
|
||||
with open(os.path.join(args.output_dir, f"{args.instance_prompt}_lora_config.json"), "w") as f:
|
||||
json.dump(lora_config, f)
|
||||
else:
|
||||
pipeline = DiffusionPipeline.from_pretrained(
|
||||
args.pretrained_model_name_or_path,
|
||||
unet=accelerator.unwrap_model(unet),
|
||||
text_encoder=accelerator.unwrap_model(text_encoder),
|
||||
revision=args.revision,
|
||||
)
|
||||
pipeline.save_pretrained(args.output_dir)
|
||||
|
||||
accelerator.end_training()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
main(args)
|
||||
+23
-13
@@ -1,7 +1,7 @@
|
||||
# todo
|
||||
from enum import Enum
|
||||
import math
|
||||
import warnings
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from typing import List, Optional
|
||||
|
||||
import torch
|
||||
@@ -76,28 +76,28 @@ class LoRAModel(torch.nn.Module):
|
||||
|
||||
def __init__(self, config, model):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.pet_config = config
|
||||
self.model = model
|
||||
self._find_and_replace()
|
||||
mark_only_lora_as_trainable(self.model, self.config.bias)
|
||||
mark_only_lora_as_trainable(self.model, self.pet_config.bias)
|
||||
|
||||
def _find_and_replace(self):
|
||||
kwargs = {
|
||||
"r": self.config.r,
|
||||
"lora_alpha": self.config.lora_alpha,
|
||||
"lora_dropout": self.config.lora_dropout,
|
||||
"fan_in_fan_out": self.config.fan_in_fan_out,
|
||||
"merge_weights": self.config.merge_weights,
|
||||
"r": self.pet_config.r,
|
||||
"lora_alpha": self.pet_config.lora_alpha,
|
||||
"lora_dropout": self.pet_config.lora_dropout,
|
||||
"fan_in_fan_out": self.pet_config.fan_in_fan_out,
|
||||
"merge_weights": self.pet_config.merge_weights,
|
||||
}
|
||||
key_list = [key for key, _ in self.model.named_modules()]
|
||||
for key in key_list:
|
||||
if any(key.endswith(target_key) for target_key in self.config.target_modules):
|
||||
if any(key.endswith(target_key) for target_key in self.pet_config.target_modules):
|
||||
parent, target, target_name = self._get_submodules(key)
|
||||
bias = target.bias is not None
|
||||
if isinstance(target, torch.nn.Linear) and self.config.enable_lora is None:
|
||||
if isinstance(target, torch.nn.Linear) and self.pet_config.enable_lora is None:
|
||||
new_module = Linear(target.in_features, target.out_features, bias=bias, **kwargs)
|
||||
elif self.config.enable_lora is not None:
|
||||
kwargs.update({"enable_lora": self.config.enable_lora})
|
||||
elif self.pet_config.enable_lora is not None:
|
||||
kwargs.update({"enable_lora": self.pet_config.enable_lora})
|
||||
if isinstance(target, Conv1D):
|
||||
in_features, out_features = target.weight.shape
|
||||
else:
|
||||
@@ -133,6 +133,16 @@ class LoRAModel(torch.nn.Module):
|
||||
except AttributeError:
|
||||
return getattr(self.model, name)
|
||||
|
||||
@property
|
||||
def modules_to_save(self):
|
||||
return None
|
||||
|
||||
def get_pet_config_as_dict(self, inference: bool = False):
|
||||
config = {k: v.value if isinstance(v, Enum) else v for k, v in asdict(self.pet_config).items()}
|
||||
if inference:
|
||||
config["inference_mode"] = True
|
||||
return config
|
||||
|
||||
|
||||
# Below code is based on https://github.com/microsoft/LoRA/blob/main/loralib/layers.py
|
||||
# and modified to work with PyTorch FSDP
|
||||
|
||||
Reference in new issue
Block a user