wassname/NALU-pytorch
1
0
Fork 0
mirror of https://github.com/wassname/NALU-pytorch.git synced 2026-08-11 05:38:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

updates

Browse Source
This commit is contained in:
Kevin
2018-08-04 04:24:14 -07:00
parent 92b0f2e9c3
commit 940041d06f
7 changed files with 62 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+8 -7
View File
@@ -43,10 +43,11 @@ python function_learning.py
```
This should generate a text file called `interpolation.txt` with the following results. (Currently only supports interpolation, I'm working on the rest)
| | Relu6 | None | NAC | NALU |
|-------|----------|----------|----------|--------|
| a + b | 0.002 | 0.004 | 0.001 | 0.017 |
| a - b | 0.046 | 0.005 | 0.000 | 0.003 |
| a * b | 83.012 | 0.444 | 5.218 | 5.218 |
| a / b | 106.441 | 0.338 | 2.096 | 2.096 |
| a ^ 2 | 94.103 | 0.630 | 3.871 | 0.196 |
| | Relu6 | None | NAC | NALU |
|---------|----------|----------|----------|--------|
| a + b | 4.472 | 0.132 | 0.154 | 0.157 |
| a - b | 85.727 | 2.224 | 2.403 | 34.610 |
| a * b | 89.257 | 4.573 | 5.382 | 1.236 |
| a / b | 97.070 | 60.594 | 5.730 | 3.042 |
| a ^ 2 | 89.987 | 2.977 | 4.718 | 1.117 |
| sqrt(a) | 5.939 | 40.243 | 7.263 | 1.119 |
function_learning.py
+2 -2
View File
@@ -13,7 +13,7 @@ NORMALIZE = True
NUM_LAYERS = 2
HIDDEN_DIM = 2
LEARNING_RATE = 1e-3
NUM_ITERS = int(7e4)
NUM_ITERS = int(1e5)
RANGE = [5, 10]
ARITHMETIC_FUNCTIONS = {
'add': lambda x, y: x + y,
@@ -120,7 +120,7 @@ def main():
# others
for net in models:
optim = torch.optim.Adam(net.parameters(), lr=LEARNING_RATE)
optim = torch.optim.RMSprop(net.parameters(), lr=LEARNING_RATE)
train(net, optim, X_train, y_train, NUM_ITERS)
mse = test(net, X_test, y_test).mean().item()
results[fn_str].append(mse)
models/mlp.py
+3 -31
View File
@@ -1,41 +1,13 @@
import math
import torch.nn as nn
from .utils import str2act
class MLP(nn.Module):
def __init__(self, activation, input_dim=1, encoding_dim=8):
super().__init__()
if activation is 'hardtanh':
self.activation = nn.Hardtanh()
elif activation is 'sigmoid':
self.activation = nn.Sigmoid()
elif activation is 'relu6':
self.activation = nn.ReLU6()
elif activation is 'tanh':
self.activation = nn.Tanh()
elif activation is 'tanhshrink':
self.activation = nn.Tanhshrink()
elif activation is 'hardshrink':
self.activation = nn.Hardshrink()
elif activation is 'leakyrelu':
self.activation = nn.LeakyReLU()
elif activation is 'softshrink':
self.activation = nn.Softshrink()
elif activation is 'softsign':
self.activation = nn.Softsign()
elif activation is 'relu':
self.activation = nn.ReLU()
elif activation is 'prelu':
self.activation = nn.PReLU()
elif activation is 'softplus':
self.activation = nn.Softplus()
elif activation is 'elu':
self.activation = nn.ELU()
elif activation is 'selu':
self.activation = nn.SELU()
else:
raise ValueError("[!] Invalid activation function.")
self.activation = str2act(activation)
self.i2h = nn.Linear(input_dim, encoding_dim)
self.h2h1 = nn.Linear(encoding_dim, encoding_dim)
models/models.py
+2 -34
View File
@@ -4,6 +4,7 @@ import torch.nn as nn
from .nac import NAC
from .nalu import NALU
from .utils import str2act
class MultiLayerNet(nn.Module):
@@ -13,40 +14,7 @@ class MultiLayerNet(nn.Module):
self.in_dim = in_dim
self.hidden_dim = hidden_dim
self.out_dim = out_dim
if activation is 'none':
self.activation = None
elif activation is 'hardtanh':
self.activation = nn.Hardtanh()
elif activation is 'sigmoid':
self.activation = nn.Sigmoid()
elif activation is 'relu6':
self.activation = nn.ReLU6()
elif activation is 'tanh':
self.activation = nn.Tanh()
elif activation is 'tanhshrink':
self.activation = nn.Tanhshrink()
elif activation is 'hardshrink':
self.activation = nn.Hardshrink()
elif activation is 'leakyrelu':
self.activation = nn.LeakyReLU()
elif activation is 'softshrink':
self.activation = nn.Softshrink()
elif activation is 'softsign':
self.activation = nn.Softsign()
elif activation is 'relu':
self.activation = nn.ReLU()
elif activation is 'prelu':
self.activation = nn.PReLU()
elif activation is 'softplus':
self.activation = nn.Softplus()
elif activation is 'elu':
self.activation = nn.ELU()
elif activation is 'selu':
self.activation = nn.SELU()
else:
raise ValueError("[!] Invalid activation function.")
self.activation = str2act(activation)
layers = []
if self.activation is not None:
models/nalu.py
+3 -2
View File
@@ -29,6 +29,7 @@ class NALU(nn.Module):
self.G = Parameter(torch.Tensor(out_features, in_features))
self.W = Parameter(torch.Tensor(out_features, in_features))
self.register_parameter('bias', None)
self.nac = NAC(in_features, out_features)
init.kaiming_uniform_(self.G, a=math.sqrt(5))
@@ -36,10 +37,10 @@ class NALU(nn.Module):
def forward(self, input):
a = self.nac(input)
g = F.sigmoid(F.linear(input, self.G, None))
g = F.sigmoid(F.linear(input, self.G, self.bias))
add_sub = g * a
log_input = torch.log(torch.abs(input) + self.eps)
m = torch.exp(F.linear(log_input, self.W, None))
m = torch.exp(F.linear(log_input, self.W, self.bias))
mul_div = (1 - g) * m
y = add_sub + mul_div
return y
models/utils.py
+37
View File
@@ -0,0 +1,37 @@
import torch
import torch.nn as nn
def str2act(s):
if s is 'none':
return None
elif s is 'hardtanh':
return nn.Hardtanh()
elif s is 'sigmoid':
return nn.Sigmoid()
elif s is 'relu6':
return nn.ReLU6()
elif s is 'tanh':
return nn.Tanh()
elif s is 'tanhshrink':
return nn.Tanhshrink()
elif s is 'hardshrink':
return nn.Hardshrink()
elif s is 'leakyrelu':
return nn.LeakyReLU()
elif s is 'softshrink':
return nn.Softshrink()
elif s is 'softsign':
return nn.Softsign()
elif s is 'relu':
return nn.ReLU()
elif s is 'prelu':
return nn.PReLU()
elif s is 'softplus':
return nn.Softplus()
elif s is 'elu':
return nn.ELU()
elif s is 'selu':
return nn.SELU()
else:
raise ValueError("[!] Invalid activation function.")
results/interpolation.txt
+7
View File
@@ -0,0 +1,7 @@
Relu6 None NAC NALU
4.472 0.132 0.154 0.157
85.727 2.224 2.403 34.610
89.257 4.573 5.382 1.236
97.070 60.594 5.730 3.042
89.987 2.977 4.718 1.117
5.939 40.243 7.263 1.119