successfully did some basic math and clipped the predicted x0 intermediate for the continuous time case

README.md

@@ -121,13 +121,3 @@ Samples and model checkpoints will be logged to `./results` periodically
     url         = {https://openreview.net/forum?id=2LdBqxc1Yv}
 }
 ```
-
-```bibtex
-@article{Choi2022PerceptionPT,
-    title   = {Perception Prioritized Training of Diffusion Models},
-    author  = {Jooyoung Choi and Jungbeom Lee and Chaehun Shin and Sungwon Kim and Hyunwoo J. Kim and Sung-Hoon Yoon},
-    journal = {ArXiv},
-    year    = {2022},
-    volume  = {abs/2204.00227}
-}
-```

denoising_diffusion_pytorch/continuous_time_gaussian_diffusion.py

@@ -5,7 +5,7 @@ import torch.nn.functional as F
 from torch.special import expm1
 
 from tqdm import tqdm
-from einops import rearrange, repeat, reduce
+from einops import rearrange, repeat
 from einops.layers.torch import Rearrange
 
 # helpers
@@ -59,14 +59,11 @@ class MonotonicLinear(nn.Module):
 
 # log(snr) that approximates the original linear schedule
 
-def log(t, eps = 1e-20):
-    return torch.log(t.clamp(min = eps))
-
 def beta_linear_log_snr(t):
-    return -log(expm1(1e-4 + 10 * (t ** 2)))
+    return -torch.log(expm1(1e-4 + 10 * (t ** 2)))
 
-def alpha_cosine_log_snr(t, s = 0.008):
-    return -log((torch.cos((t + s) / (1 + s) * torch.pi * 0.5) ** -2) - 1, eps = 1e-5)
+def alpha_cosine_log_snr(t):
+    raise NotImplementedError
 
 class learned_noise_schedule(nn.Module):
     """ described in section H and then I.2 of the supplementary material for variational ddpm paper """
@@ -120,9 +117,7 @@ class ContinuousTimeGaussianDiffusion(nn.Module):
         num_sample_steps = 500,
         clip_sample_denoised = True,
         learned_schedule_net_hidden_dim = 1024,
-        learned_noise_schedule_frac_gradient = 1.,   # between 0 and 1, determines what percentage of gradients go back, so one can update the learned noise schedule more slowly
-        p2_loss_weight_gamma = 0.,                   # p2 loss weight, from https://arxiv.org/abs/2204.00227 - 0 is equivalent to weight of 1 across time
-        p2_loss_weight_k = 1
+        learned_noise_schedule_frac_gradient = 1.   # between 0 and 1, determines what percentage of gradients go back, so one can update the learned noise schedule more slowly
     ):
         super().__init__()
         assert not denoise_fn.sinusoidal_cond_mlp
@@ -140,8 +135,6 @@ class ContinuousTimeGaussianDiffusion(nn.Module):
 
         if noise_schedule == 'linear':
             self.log_snr = beta_linear_log_snr
-        elif noise_schedule == 'cosine':
-            self.log_snr = alpha_cosine_log_snr
         elif noise_schedule == 'learned':
             log_snr_max, log_snr_min = [beta_linear_log_snr(torch.tensor([time])).item() for time in (0., 1.)]
 
@@ -159,14 +152,6 @@ class ContinuousTimeGaussianDiffusion(nn.Module):
         self.num_sample_steps = num_sample_steps
         self.clip_sample_denoised = clip_sample_denoised
 
-        # p2 loss weight
-        # proposed https://arxiv.org/abs/2204.00227
-
-        assert p2_loss_weight_gamma <= 2, 'in paper, they noticed any gamma greater than 2 is harmful'
-
-        self.p2_loss_weight_gamma = p2_loss_weight_gamma  # recommended to be 0.5 or 1
-        self.p2_loss_weight_k = p2_loss_weight_k
-
     @property
     def device(self):
         return next(self.denoise_fn.parameters()).device
@@ -202,7 +187,7 @@ class ContinuousTimeGaussianDiffusion(nn.Module):
             # in Imagen, this was changed to dynamic thresholding
             x_start.clamp_(-1., 1.)
 
-            model_mean = alpha_next * (x * (1 - c) / alpha + c * x_start)
+            model_mean = alpha_next / alpha * x * (1 - c) + alpha_next * c * x_start
         else:
             model_mean = alpha_next / alpha * (x - c * sigma * pred_noise)
 
@@ -265,17 +250,9 @@ class ContinuousTimeGaussianDiffusion(nn.Module):
         noise = default(noise, lambda: torch.randn_like(x_start))
 
         x, log_snr = self.q_sample(x_start = x_start, times = times, noise = noise)
+
         model_out = self.denoise_fn(x, log_snr)
-
-        losses = self.loss_fn(model_out, noise, reduction = 'none')
-        losses = reduce(losses, 'b ... -> b', 'mean')
-
-        if self.p2_loss_weight_gamma >= 0:
-            # following eq 8. in https://arxiv.org/abs/2204.00227
-            loss_weight = (self.p2_loss_weight_k + log_snr.exp()) ** -self.p2_loss_weight_gamma
-            losses = losses * loss_weight
-
-        return losses.mean()
+        return self.loss_fn(model_out, noise)
 
     def forward(self, img, *args, **kwargs):
         b, c, h, w, device, img_size, = *img.shape, img.device, self.image_size

denoising_diffusion_pytorch/denoising_diffusion_pytorch.py

@@ -7,7 +7,6 @@ from inspect import isfunction
 from functools import partial
 
 from torch.utils import data
-from multiprocessing import cpu_count
 from torch.cuda.amp import autocast, GradScaler
 
 from pathlib import Path
@@ -542,7 +541,7 @@ class GaussianDiffusion(nn.Module):
 # dataset classes
 
 class Dataset(data.Dataset):
-    def __init__(self, folder, image_size, exts = ['jpg', 'jpeg', 'png'], augment_horizontal_flip = False):
+    def __init__(self, folder, image_size, exts = ['jpg', 'jpeg', 'png']):
         super().__init__()
         self.folder = folder
         self.image_size = image_size
@@ -550,7 +549,7 @@ class Dataset(data.Dataset):
 
         self.transform = transforms.Compose([
             transforms.Resize(image_size),
-            transforms.RandomHorizontalFlip() if augment_horizontal_flip else nn.Identity(),
+            transforms.RandomHorizontalFlip(),
             transforms.CenterCrop(image_size),
             transforms.ToTensor()
         ])
@@ -581,8 +580,7 @@ class Trainer(object):
         step_start_ema = 2000,
         update_ema_every = 10,
         save_and_sample_every = 1000,
-        results_folder = './results',
-        augment_horizontal_flip = True
+        results_folder = './results'
     ):
         super().__init__()
         self.model = diffusion_model
@@ -598,8 +596,8 @@ class Trainer(object):
         self.gradient_accumulate_every = gradient_accumulate_every
         self.train_num_steps = train_num_steps
 
-        self.ds = Dataset(folder, image_size, augment_horizontal_flip = augment_horizontal_flip)
-        self.dl = cycle(data.DataLoader(self.ds, batch_size = train_batch_size, shuffle = True, pin_memory = True, num_workers = cpu_count()))
+        self.ds = Dataset(folder, image_size)
+        self.dl = cycle(data.DataLoader(self.ds, batch_size = train_batch_size, shuffle=True, pin_memory=True))
         self.opt = Adam(diffusion_model.parameters(), lr=train_lr)
 
         self.step = 0

setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 setup(
   name = 'denoising-diffusion-pytorch',
   packages = find_packages(),
-  version = '0.18.3',
+  version = '0.17.5',
   license='MIT',
   description = 'Denoising Diffusion Probabilistic Models - Pytorch',
   author = 'Phil Wang',