more skip connections, as in guided diffusion

denoising_diffusion_pytorch/continuous_time_gaussian_diffusion.py

@@ -1,3 +1,4 @@
+import math
 import torch
 from torch import sqrt
 from torch import nn, einsum
@@ -66,7 +67,7 @@ def beta_linear_log_snr(t):
     return -log(expm1(1e-4 + 10 * (t ** 2)))
 
 def alpha_cosine_log_snr(t, s = 0.008):
-    return -log((torch.cos((t + s) / (1 + s) * torch.pi * 0.5) ** -2) - 1, eps = 1e-5)
+    return -log((torch.cos((t + s) / (1 + s) * math.pi * 0.5) ** -2) - 1, eps = 1e-5)
 
 class learned_noise_schedule(nn.Module):
     """ described in section H and then I.2 of the supplementary material for variational ddpm paper """

denoising_diffusion_pytorch/denoising_diffusion_pytorch.py

@@ -60,11 +60,14 @@ class Residual(nn.Module):
     def forward(self, x, *args, **kwargs):
         return self.fn(x, *args, **kwargs) + x
 
-def Upsample(dim):
+def Upsample(dim, dim_out = None):
-    return nn.ConvTranspose2d(dim, dim, 4, 2, 1)
+    return nn.Sequential(
+        nn.Upsample(scale_factor = 2, mode = 'nearest'),
+        nn.Conv2d(dim, default(dim_out, dim), 3, padding = 1)
+    )
 
-def Downsample(dim):
+def Downsample(dim, dim_out = None):
-    return nn.Conv2d(dim, dim, 4, 2, 1)
+    return nn.Conv2d(dim, default(dim_out, dim), 4, 2, 1)
 
 class LayerNorm(nn.Module):
     def __init__(self, dim, eps = 1e-5):
@@ -277,10 +280,10 @@ class Unet(nn.Module):
             is_last = ind >= (num_resolutions - 1)
 
             self.downs.append(nn.ModuleList([
-                block_klass(dim_in, dim_out, time_emb_dim = time_dim),
+                block_klass(dim_in, dim_in, time_emb_dim = time_dim),
-                block_klass(dim_out, dim_out, time_emb_dim = time_dim),
+                block_klass(dim_in, dim_in, time_emb_dim = time_dim),
-                Residual(PreNorm(dim_out, LinearAttention(dim_out))),
+                Residual(PreNorm(dim_in, LinearAttention(dim_in))),
-                Downsample(dim_out) if not is_last else nn.Identity()
+                Downsample(dim_in, dim_out) if not is_last else nn.Conv2d(dim_in, dim_out, 3, padding = 1)
             ]))
 
         mid_dim = dims[-1]
@@ -292,10 +295,10 @@ class Unet(nn.Module):
             is_last = ind == (len(in_out) - 1)
 
             self.ups.append(nn.ModuleList([
-                block_klass(dim_out * 2, dim_in, time_emb_dim = time_dim),
+                block_klass(dim_out + dim_in, dim_out, time_emb_dim = time_dim),
-                block_klass(dim_in, dim_in, time_emb_dim = time_dim),
+                block_klass(dim_out + dim_in, dim_out, time_emb_dim = time_dim),
-                Residual(PreNorm(dim_in, LinearAttention(dim_in))),
+                Residual(PreNorm(dim_out, LinearAttention(dim_out))),
-                Upsample(dim_in) if not is_last else nn.Identity()
+                Upsample(dim_out, dim_in) if not is_last else  nn.Conv2d(dim_out, dim_in, 3, padding = 1)
             ]))
 
         default_out_dim = channels * (1 if not learned_variance else 2)
@@ -314,9 +317,12 @@ class Unet(nn.Module):
 
         for block1, block2, attn, downsample in self.downs:
             x = block1(x, t)
+            h.append(x)
+
             x = block2(x, t)
             x = attn(x)
             h.append(x)
+
             x = downsample(x)
 
         x = self.mid_block1(x, t)
@@ -326,8 +332,11 @@ class Unet(nn.Module):
         for block1, block2, attn, upsample in self.ups:
             x = torch.cat((x, h.pop()), dim = 1)
             x = block1(x, t)
+
+            x = torch.cat((x, h.pop()), dim = 1)
             x = block2(x, t)
             x = attn(x)
+
             x = upsample(x)
 
         x = torch.cat((x, r), dim = 1)
@@ -355,7 +364,7 @@ def cosine_beta_schedule(timesteps, s = 0.008):
     """
     steps = timesteps + 1
     x = torch.linspace(0, timesteps, steps, dtype = torch.float64)
-    alphas_cumprod = torch.cos(((x / timesteps) + s) / (1 + s) * torch.pi * 0.5) ** 2
+    alphas_cumprod = torch.cos(((x / timesteps) + s) / (1 + s) * math.pi * 0.5) ** 2
     alphas_cumprod = alphas_cumprod / alphas_cumprod[0]
     betas = 1 - (alphas_cumprod[1:] / alphas_cumprod[:-1])
     return torch.clip(betas, 0, 0.999)
@@ -590,7 +599,7 @@ class Trainer(object):
         gradient_accumulate_every = 2,
         amp = False,
         step_start_ema = 2000,
-        update_ema_every = 10,
+        ema_update_every = 10,
         save_and_sample_every = 1000,
         results_folder = './results',
         augment_horizontal_flip = True
@@ -599,8 +608,7 @@ class Trainer(object):
         self.image_size = diffusion_model.image_size
 
         self.model = diffusion_model
-        self.ema = EMA(diffusion_model, beta = ema_decay)
+        self.ema = EMA(diffusion_model, beta = ema_decay, update_every = ema_update_every)
-        self.update_ema_every = update_ema_every
 
         self.step_start_ema = step_start_ema
         self.save_and_sample_every = save_and_sample_every

setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 setup(
   name = 'denoising-diffusion-pytorch',
   packages = find_packages(),
-  version = '0.21.0',
+  version = '0.22.0',
   license='MIT',
   description = 'Denoising Diffusion Probabilistic Models - Pytorch',
   author = 'Phil Wang',