From 940041d06fb7ac1c2c0f6fa0c51644b121c895a2 Mon Sep 17 00:00:00 2001 From: Kevin Date: Sat, 4 Aug 2018 04:24:14 -0700 Subject: [PATCH] updates --- README.md | 15 ++++++++------- function_learning.py | 4 ++-- models/mlp.py | 34 +++------------------------------- models/models.py | 36 ++---------------------------------- models/nalu.py | 5 +++-- models/utils.py | 37 +++++++++++++++++++++++++++++++++++++ results/interpolation.txt | 7 +++++++ 7 files changed, 62 insertions(+), 76 deletions(-) create mode 100644 models/utils.py create mode 100644 results/interpolation.txt diff --git a/README.md b/README.md index a89a8ca..867e4b7 100644 --- a/README.md +++ b/README.md @@ -43,10 +43,11 @@ python function_learning.py ``` This should generate a text file called `interpolation.txt` with the following results. (Currently only supports interpolation, I'm working on the rest) -| | Relu6 | None | NAC | NALU | -|-------|----------|----------|----------|--------| -| a + b | 0.002 | 0.004 | 0.001 | 0.017 | -| a - b | 0.046 | 0.005 | 0.000 | 0.003 | -| a * b | 83.012 | 0.444 | 5.218 | 5.218 | -| a / b | 106.441 | 0.338 | 2.096 | 2.096 | -| a ^ 2 | 94.103 | 0.630 | 3.871 | 0.196 | +| | Relu6 | None | NAC | NALU | +|---------|----------|----------|----------|--------| +| a + b | 4.472 | 0.132 | 0.154 | 0.157 | +| a - b | 85.727 | 2.224 | 2.403 | 34.610 | +| a * b | 89.257 | 4.573 | 5.382 | 1.236 | +| a / b | 97.070 | 60.594 | 5.730 | 3.042 | +| a ^ 2 | 89.987 | 2.977 | 4.718 | 1.117 | +| sqrt(a) | 5.939 | 40.243 | 7.263 | 1.119 | diff --git a/function_learning.py b/function_learning.py index 3cb8b69..d8ddcfc 100644 --- a/function_learning.py +++ b/function_learning.py @@ -13,7 +13,7 @@ NORMALIZE = True NUM_LAYERS = 2 HIDDEN_DIM = 2 LEARNING_RATE = 1e-3 -NUM_ITERS = int(7e4) +NUM_ITERS = int(1e5) RANGE = [5, 10] ARITHMETIC_FUNCTIONS = { 'add': lambda x, y: x + y, @@ -120,7 +120,7 @@ def main(): # others for net in models: - optim = torch.optim.Adam(net.parameters(), lr=LEARNING_RATE) + optim = torch.optim.RMSprop(net.parameters(), lr=LEARNING_RATE) train(net, optim, X_train, y_train, NUM_ITERS) mse = test(net, X_test, y_test).mean().item() results[fn_str].append(mse) diff --git a/models/mlp.py b/models/mlp.py index 137d843..7078527 100644 --- a/models/mlp.py +++ b/models/mlp.py @@ -1,41 +1,13 @@ import math import torch.nn as nn +from .utils import str2act + class MLP(nn.Module): def __init__(self, activation, input_dim=1, encoding_dim=8): super().__init__() - - if activation is 'hardtanh': - self.activation = nn.Hardtanh() - elif activation is 'sigmoid': - self.activation = nn.Sigmoid() - elif activation is 'relu6': - self.activation = nn.ReLU6() - elif activation is 'tanh': - self.activation = nn.Tanh() - elif activation is 'tanhshrink': - self.activation = nn.Tanhshrink() - elif activation is 'hardshrink': - self.activation = nn.Hardshrink() - elif activation is 'leakyrelu': - self.activation = nn.LeakyReLU() - elif activation is 'softshrink': - self.activation = nn.Softshrink() - elif activation is 'softsign': - self.activation = nn.Softsign() - elif activation is 'relu': - self.activation = nn.ReLU() - elif activation is 'prelu': - self.activation = nn.PReLU() - elif activation is 'softplus': - self.activation = nn.Softplus() - elif activation is 'elu': - self.activation = nn.ELU() - elif activation is 'selu': - self.activation = nn.SELU() - else: - raise ValueError("[!] Invalid activation function.") + self.activation = str2act(activation) self.i2h = nn.Linear(input_dim, encoding_dim) self.h2h1 = nn.Linear(encoding_dim, encoding_dim) diff --git a/models/models.py b/models/models.py index 25f4349..e43eea9 100644 --- a/models/models.py +++ b/models/models.py @@ -4,6 +4,7 @@ import torch.nn as nn from .nac import NAC from .nalu import NALU +from .utils import str2act class MultiLayerNet(nn.Module): @@ -13,40 +14,7 @@ class MultiLayerNet(nn.Module): self.in_dim = in_dim self.hidden_dim = hidden_dim self.out_dim = out_dim - - if activation is 'none': - self.activation = None - elif activation is 'hardtanh': - self.activation = nn.Hardtanh() - elif activation is 'sigmoid': - self.activation = nn.Sigmoid() - elif activation is 'relu6': - self.activation = nn.ReLU6() - elif activation is 'tanh': - self.activation = nn.Tanh() - elif activation is 'tanhshrink': - self.activation = nn.Tanhshrink() - elif activation is 'hardshrink': - self.activation = nn.Hardshrink() - elif activation is 'leakyrelu': - self.activation = nn.LeakyReLU() - elif activation is 'softshrink': - self.activation = nn.Softshrink() - elif activation is 'softsign': - self.activation = nn.Softsign() - elif activation is 'relu': - self.activation = nn.ReLU() - elif activation is 'prelu': - self.activation = nn.PReLU() - elif activation is 'softplus': - self.activation = nn.Softplus() - elif activation is 'elu': - self.activation = nn.ELU() - elif activation is 'selu': - self.activation = nn.SELU() - else: - raise ValueError("[!] Invalid activation function.") - + self.activation = str2act(activation) layers = [] if self.activation is not None: diff --git a/models/nalu.py b/models/nalu.py index 3bcdc7c..fd4e6f3 100644 --- a/models/nalu.py +++ b/models/nalu.py @@ -29,6 +29,7 @@ class NALU(nn.Module): self.G = Parameter(torch.Tensor(out_features, in_features)) self.W = Parameter(torch.Tensor(out_features, in_features)) + self.register_parameter('bias', None) self.nac = NAC(in_features, out_features) init.kaiming_uniform_(self.G, a=math.sqrt(5)) @@ -36,10 +37,10 @@ class NALU(nn.Module): def forward(self, input): a = self.nac(input) - g = F.sigmoid(F.linear(input, self.G, None)) + g = F.sigmoid(F.linear(input, self.G, self.bias)) add_sub = g * a log_input = torch.log(torch.abs(input) + self.eps) - m = torch.exp(F.linear(log_input, self.W, None)) + m = torch.exp(F.linear(log_input, self.W, self.bias)) mul_div = (1 - g) * m y = add_sub + mul_div return y diff --git a/models/utils.py b/models/utils.py new file mode 100644 index 0000000..f688350 --- /dev/null +++ b/models/utils.py @@ -0,0 +1,37 @@ +import torch +import torch.nn as nn + + +def str2act(s): + if s is 'none': + return None + elif s is 'hardtanh': + return nn.Hardtanh() + elif s is 'sigmoid': + return nn.Sigmoid() + elif s is 'relu6': + return nn.ReLU6() + elif s is 'tanh': + return nn.Tanh() + elif s is 'tanhshrink': + return nn.Tanhshrink() + elif s is 'hardshrink': + return nn.Hardshrink() + elif s is 'leakyrelu': + return nn.LeakyReLU() + elif s is 'softshrink': + return nn.Softshrink() + elif s is 'softsign': + return nn.Softsign() + elif s is 'relu': + return nn.ReLU() + elif s is 'prelu': + return nn.PReLU() + elif s is 'softplus': + return nn.Softplus() + elif s is 'elu': + return nn.ELU() + elif s is 'selu': + return nn.SELU() + else: + raise ValueError("[!] Invalid activation function.") diff --git a/results/interpolation.txt b/results/interpolation.txt new file mode 100644 index 0000000..c98c3ee --- /dev/null +++ b/results/interpolation.txt @@ -0,0 +1,7 @@ +Relu6 None NAC NALU +4.472 0.132 0.154 0.157 +85.727 2.224 2.403 34.610 +89.257 4.573 5.382 1.236 +97.070 60.594 5.730 3.042 +89.987 2.977 4.718 1.117 +5.939 40.243 7.263 1.119