From 7b51e30da7710cbd277ad32455377b431ab3a968 Mon Sep 17 00:00:00 2001 From: Phil Wang Date: Tue, 24 Aug 2021 14:28:15 -0700 Subject: [PATCH] fix layernorm --- .../denoising_diffusion_pytorch.py | 14 +++++++++++++- setup.py | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/denoising_diffusion_pytorch/denoising_diffusion_pytorch.py b/denoising_diffusion_pytorch/denoising_diffusion_pytorch.py index 99b0cd1..8ca6788 100644 --- a/denoising_diffusion_pytorch/denoising_diffusion_pytorch.py +++ b/denoising_diffusion_pytorch/denoising_diffusion_pytorch.py @@ -111,11 +111,23 @@ class Downsample(nn.Module): def forward(self, x): return self.conv(x) +class LayerNorm(nn.Module): + def __init__(self, dim, eps = 1e-5): + super().__init__() + self.eps = eps + self.g = nn.Parameter(torch.ones(1, dim, 1, 1)) + self.b = nn.Parameter(torch.zeros(1, dim, 1, 1)) + + def forward(self, x): + std = torch.var(x, dim = 1, unbiased = False, keepdim = True).sqrt() + mean = torch.mean(x, dim = 1, keepdim = True) + return (x - mean) / (std + self.eps) * self.g + self.b + class PreNorm(nn.Module): def __init__(self, dim, fn): super().__init__() self.fn = fn - self.norm = nn.InstanceNorm2d(dim, affine = True) + self.norm = LayerNorm(dim) def forward(self, x): x = self.norm(x) diff --git a/setup.py b/setup.py index 02a428e..9749e76 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages setup( name = 'denoising-diffusion-pytorch', packages = find_packages(), - version = '0.6.8', + version = '0.6.9', license='MIT', description = 'Denoising Diffusion Probabilistic Models - Pytorch', author = 'Phil Wang',