From eba44498d1b33749df6e183bdc0899cf2918a5f3 Mon Sep 17 00:00:00 2001 From: Phil Wang Date: Fri, 29 Jul 2022 13:29:52 -0700 Subject: [PATCH] higher epsilon for fp16 in layernorm --- denoising_diffusion_pytorch/denoising_diffusion_pytorch.py | 7 +++---- setup.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/denoising_diffusion_pytorch/denoising_diffusion_pytorch.py b/denoising_diffusion_pytorch/denoising_diffusion_pytorch.py index 9479ca5..d9a73bb 100644 --- a/denoising_diffusion_pytorch/denoising_diffusion_pytorch.py +++ b/denoising_diffusion_pytorch/denoising_diffusion_pytorch.py @@ -89,16 +89,15 @@ def Downsample(dim, dim_out = None): return nn.Conv2d(dim, default(dim_out, dim), 4, 2, 1) class LayerNorm(nn.Module): - def __init__(self, dim, eps = 1e-5): + def __init__(self, dim): super().__init__() - self.eps = eps self.g = nn.Parameter(torch.ones(1, dim, 1, 1)) - self.b = nn.Parameter(torch.zeros(1, dim, 1, 1)) def forward(self, x): + eps = 1e-5 if x.dtype == torch.float32 else 1e-3 var = torch.var(x, dim = 1, unbiased = False, keepdim = True) mean = torch.mean(x, dim = 1, keepdim = True) - return (x - mean) / (var + self.eps).sqrt() * self.g + self.b + return (x - mean) * (var + eps).rsqrt() * self.g class PreNorm(nn.Module): def __init__(self, dim, fn): diff --git a/setup.py b/setup.py index 0eb0bf7..eec0773 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages setup( name = 'denoising-diffusion-pytorch', packages = find_packages(), - version = '0.26.4', + version = '0.26.5', license='MIT', description = 'Denoising Diffusion Probabilistic Models - Pytorch', author = 'Phil Wang',