add the v-parameterization from Salimans et al for the discrete case, allow for distillation

denoising_diffusion_pytorch/continuous_time_gaussian_diffusion.py

@@ -126,7 +126,7 @@ class ContinuousTimeGaussianDiffusion(nn.Module):
         p2_loss_weight_k = 1
     ):
         super().__init__()
-        assert model.learned_sinusoidal_cond
+        assert model.random_or_learned_sinusoidal_cond
         assert not model.self_condition, 'not supported yet'
 
         self.model = model

denoising_diffusion_pytorch/denoising_diffusion_pytorch.py

@@ -140,15 +140,15 @@ class SinusoidalPosEmb(nn.Module):
         emb = torch.cat((emb.sin(), emb.cos()), dim=-1)
         return emb
 
-class LearnedSinusoidalPosEmb(nn.Module):
+class RandomOrLearnedSinusoidalPosEmb(nn.Module):
-    """ following @crowsonkb 's lead with learned sinusoidal pos emb """
+    """ following @crowsonkb 's lead with random (learned optional) sinusoidal pos emb """
     """ https://github.com/crowsonkb/v-diffusion-jax/blob/master/diffusion/models/danbooru_128.py#L8 """
 
-    def __init__(self, dim):
+    def __init__(self, dim, is_random = False):
         super().__init__()
         assert (dim % 2) == 0
         half_dim = dim // 2
-        self.weights = nn.Parameter(torch.randn(half_dim))
+        self.weights = nn.Parameter(torch.randn(half_dim), requires_grad = not is_random)
 
     def forward(self, x):
         x = rearrange(x, 'b -> b 1')
@@ -271,6 +271,7 @@ class Unet(nn.Module):
         resnet_block_groups = 8,
         learned_variance = False,
         learned_sinusoidal_cond = False,
+        random_fourier_features = False,
         learned_sinusoidal_dim = 16
     ):
         super().__init__()
@@ -293,10 +294,10 @@ class Unet(nn.Module):
 
         time_dim = dim * 4
 
-        self.learned_sinusoidal_cond = learned_sinusoidal_cond
+        self.random_or_learned_sinusoidal_cond = learned_sinusoidal_cond or random_fourier_features
 
-        if learned_sinusoidal_cond:
+        if self.random_or_learned_sinusoidal_cond:
-            sinu_pos_emb = LearnedSinusoidalPosEmb(learned_sinusoidal_dim)
+            sinu_pos_emb = RandomOrLearnedSinusoidalPosEmb(learned_sinusoidal_dim, random_fourier_features)
             fourier_dim = learned_sinusoidal_dim + 1
         else:
             sinu_pos_emb = SinusoidalPosEmb(dim)
@@ -429,7 +430,7 @@ class GaussianDiffusion(nn.Module):
     ):
         super().__init__()
         assert not (type(self) == GaussianDiffusion and model.channels != model.out_dim)
-        assert not model.learned_sinusoidal_cond
+        assert not model.random_or_learned_sinusoidal_cond
 
         self.model = model
         self.channels = self.model.channels
@@ -439,7 +440,7 @@ class GaussianDiffusion(nn.Module):
 
         self.objective = objective
 
-        assert objective in {'pred_noise', 'pred_x0'}, 'objective must be either pred_noise (predict noise) or pred_x0 (predict image start)'
+        assert objective in {'pred_noise', 'pred_x0', 'pred_v'}, 'objective must be either pred_noise (predict noise) or pred_x0 (predict image start) or pred_v (predict v [v-parameterization as defined in appendix D of progressive distillation paper, used in imagen-video successfully])'
 
         if beta_schedule == 'linear':
             betas = linear_beta_schedule(timesteps)
@@ -510,6 +511,18 @@ class GaussianDiffusion(nn.Module):
             extract(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape)
         )
 
+    def predict_v(self, x_start, t, noise):
+        return (
+            extract(self.sqrt_alphas_cumprod, t, x_start.shape) * noise -
+            extract(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) * x_start
+        )
+
+    def predict_start_from_v(self, x_t, t, v):
+        return (
+            extract(self.sqrt_alphas_cumprod, t, x_t.shape) * x_t -
+            extract(self.sqrt_one_minus_alphas_cumprod, t, x_t.shape) * v
+        )
+
     def q_posterior(self, x_start, x_t, t):
         posterior_mean = (
             extract(self.posterior_mean_coef1, t, x_t.shape) * x_start +
@@ -533,6 +546,12 @@ class GaussianDiffusion(nn.Module):
             x_start = maybe_clip(x_start)
             pred_noise = self.predict_noise_from_start(x, t, x_start)
 
+        elif self.objective == 'pred_v':
+            v = model_output
+            x_start = self.predict_start_from_v(x, t, v)
+            x_start = maybe_clip(x_start)
+            pred_noise = self.predict_noise_from_start(x, t, x_start)
+
         return ModelPrediction(pred_noise, x_start)
 
     def p_mean_variance(self, x, t, x_self_cond = None, clip_denoised = True):
@@ -670,6 +689,9 @@ class GaussianDiffusion(nn.Module):
             target = noise
         elif self.objective == 'pred_x0':
             target = x_start
+        elif self.objective == 'pred_v':
+            v = self.predict_v(x_start, t, noise)
+            target = v
         else:
             raise ValueError(f'unknown objective {self.objective}')
 

denoising_diffusion_pytorch/elucidated_diffusion.py

@@ -52,7 +52,7 @@ class ElucidatedDiffusion(nn.Module):
         S_noise = 1.003,
     ):
         super().__init__()
-        assert net.learned_sinusoidal_cond
+        assert net.random_or_learned_sinusoidal_cond
         self.self_condition = net.self_condition
 
         self.net = net

denoising_diffusion_pytorch/v_param_continuous_time_gaussian_diffusion.py

@@ -61,7 +61,7 @@ class VParamContinuousTimeGaussianDiffusion(nn.Module):
         clip_sample_denoised = True,
     ):
         super().__init__()
-        assert model.learned_sinusoidal_cond
+        assert model.random_or_learned_sinusoidal_cond
         assert not model.self_condition, 'not supported yet'
 
         self.model = model

setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 setup(
   name = 'denoising-diffusion-pytorch',
   packages = find_packages(),
-  version = '0.29.0',
+  version = '0.30.0',
   license='MIT',
   description = 'Denoising Diffusion Probabilistic Models - Pytorch',
   author = 'Phil Wang',