From a19331aa595ae46ce5c13d376d7f1b0d237db6cc Mon Sep 17 00:00:00 2001
From: Phil Wang <lucidrains@gmail.com>
Date: Wed, 8 Jun 2022 10:19:02 -0700
Subject: [PATCH] fix learned noise schedule

---
 .../continuous_time_gaussian_diffusion.py        | 16 ++++++++++++----
 setup.py                                         |  2 +-
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/denoising_diffusion_pytorch/continuous_time_gaussian_diffusion.py b/denoising_diffusion_pytorch/continuous_time_gaussian_diffusion.py
index 69ae277..b4fa37c 100644
--- a/denoising_diffusion_pytorch/continuous_time_gaussian_diffusion.py
+++ b/denoising_diffusion_pytorch/continuous_time_gaussian_diffusion.py
@@ -98,7 +98,7 @@ class learned_noise_schedule(nn.Module):
 
         x = self.net(x)
 
-        normalized = self.slope * ((x - out_one) / (out_zero - out_one)) + self.intercept
+        normalized = self.slope * ((x - out_zero) / (out_one - out_zero)) + self.intercept
         return normalized
 
 class ContinuousTimeGaussianDiffusion(nn.Module):
@@ -110,7 +110,8 @@ class ContinuousTimeGaussianDiffusion(nn.Module):
         channels = 3,
         loss_type = 'l1',
         noise_schedule = 'linear',
-        num_sample_steps = 500
+        num_sample_steps = 500,
+        clip_sample_after_noise = False
     ):
         super().__init__()
         assert not denoise_fn.sinusoidal_cond_mlp
@@ -142,6 +143,10 @@ class ContinuousTimeGaussianDiffusion(nn.Module):
 
         self.num_sample_steps = num_sample_steps
 
+        # clipping related hyperparameters
+
+        self.clip_sample_after_noise = clip_sample_after_noise
+
     @property
     def device(self):
         return next(self.denoise_fn.parameters()).device
@@ -203,9 +208,12 @@ class ContinuousTimeGaussianDiffusion(nn.Module):
             times_next = steps[i + 1]
             img = self.p_sample(img, times, times_next)
 
-        img.clamp_(-1., 1.)
+            if self.clip_sample_after_noise:
+                # clip after noise is added. perhaps this is sufficient for Imagen dynamic thresholding?
+                img.clamp_(-1., 1.)
+
         img = unnormalize_to_zero_to_one(img)
-        return img
+        return img.clamp(0., 1.)
 
     @torch.no_grad()
     def sample(self, batch_size = 16):
diff --git a/setup.py b/setup.py
index 5c4f1a6..46a2711 100644
--- a/setup.py
+++ b/setup.py
@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 setup(
   name = 'denoising-diffusion-pytorch',
   packages = find_packages(),
-  version = '0.17.0',
+  version = '0.17.1',
   license='MIT',
   description = 'Denoising Diffusion Probabilistic Models - Pytorch',
   author = 'Phil Wang',