From 6f9769339917dc6153afa8142aaf535b020c3fc2 Mon Sep 17 00:00:00 2001 From: josephyearsley Date: Wed, 25 Jan 2017 12:47:56 +0000 Subject: [PATCH 1/9] Added PELU Activation --- .idea/encodings.xml | 6 + .idea/keras-contrib.iml | 12 ++ .idea/misc.xml | 29 +++ .idea/modules.xml | 8 + .idea/vcs.xml | 6 + .idea/workspace.xml | 209 +++++++++++++++++++ keras_contrib/layers/advanced_activations.py | 72 +++++++ 7 files changed, 342 insertions(+) create mode 100644 .idea/encodings.xml create mode 100644 .idea/keras-contrib.iml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml create mode 100644 .idea/workspace.xml diff --git a/.idea/encodings.xml b/.idea/encodings.xml new file mode 100644 index 0000000..97626ba --- /dev/null +++ b/.idea/encodings.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/keras-contrib.iml b/.idea/keras-contrib.iml new file mode 100644 index 0000000..e98082a --- /dev/null +++ b/.idea/keras-contrib.iml @@ -0,0 +1,12 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..bc2d718 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..d4181d1 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..63feacb --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,209 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1485340173501 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/keras_contrib/layers/advanced_activations.py b/keras_contrib/layers/advanced_activations.py index 6b39244..199600b 100644 --- a/keras_contrib/layers/advanced_activations.py +++ b/keras_contrib/layers/advanced_activations.py @@ -2,3 +2,75 @@ from .. import initializations from keras.engine import Layer from .. import backend as K import numpy as np + + +class PELU(Layer): + """Parametric Exponential Linear Unit. + It follows: + `f(x) = alphas * x for x < 0`, + `f(x) = x for x >= 0`, + where `alphas` is a learned array with the same shape as x. + # Input shape + Arbitrary. Use the keyword argument `input_shape` + (tuple of integers, does not include the samples axis) + when using this layer as the first layer in a model. + # Output shape + Same shape as the input. + # Arguments + init: initialization function for the weights. + weights: initial weights, as a list of a single Numpy array. + shared_axes: the axes along which to share learnable + parameters for the activation function. + For example, if the incoming feature maps + are from a 2D convolution + with output shape `(batch, height, width, channels)`, + and you wish to share parameters across space + so that each filter only has one set of parameters, + set `shared_axes=[1, 2]`. + # References + - [PARAMETRIC EXPONENTIAL LINEAR UNIT FOR DEEP CONVOLUTIONAL NEURAL NETWORKS](https://arxiv.org/abs/1605.09332v3) + """ + + def __init__(self, init='zero', weights=None, shared_axes=None, **kwargs): + self.supports_masking = True + self.init = initializations.get(init) + self.initial_weights = weights + if not isinstance(shared_axes, (list, tuple)): + self.shared_axes = [shared_axes] + else: + self.shared_axes = list(shared_axes) + super(PELU, self).__init__(**kwargs) + + def build(self, input_shape): + param_shape = list(input_shape[1:]) + self.param_broadcast = [False] * len(param_shape) + if self.shared_axes[0] is not None: + for i in self.shared_axes: + param_shape[i - 1] = 1 + self.param_broadcast[i - 1] = True + + # Initialised as ones to emulate the default ELU + self.alphas = K.ones(param_shape, dtype=K.floatx(), + name='{}_alphas'.format(self.name)) + self.betas = K.ones(param_shape, dtype=K.floatx(), + name='{}_betas'.format(self.name)) + + self.trainable_weights = [self.alphas, self.betas] + + if self.initial_weights is not None: + self.set_weights(self.initial_weights) + del self.initial_weights + + def call(self, x, mask=None): + pos = K.relu(x) * (self.alphas / self.betas) + if K.backend() == 'theano': + neg = (K.pattern_broadcast(self.alphas, self.param_broadcast) * + (K.exp(x / self.betas) - 1)) + else: + neg = self.alphas * (K.exp(x / self.betas) - 1) + return pos + neg + + def get_config(self): + config = {'init': self.init.__name__} + base_config = super(PELU, self).get_config() + return dict(list(base_config.items()) + list(config.items())) From c0b7873086e81f8ed6ff344540c2e4a35d48d90a Mon Sep 17 00:00:00 2001 From: josephyearsley Date: Wed, 25 Jan 2017 12:48:23 +0000 Subject: [PATCH 2/9] Added PELU Activation --- .idea/encodings.xml | 6 -- .idea/misc.xml | 29 ------ .idea/modules.xml | 8 -- .idea/vcs.xml | 6 -- .idea/workspace.xml | 209 -------------------------------------------- 5 files changed, 258 deletions(-) delete mode 100644 .idea/encodings.xml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/vcs.xml delete mode 100644 .idea/workspace.xml diff --git a/.idea/encodings.xml b/.idea/encodings.xml deleted file mode 100644 index 97626ba..0000000 --- a/.idea/encodings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index bc2d718..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index d4181d1..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml deleted file mode 100644 index 63feacb..0000000 --- a/.idea/workspace.xml +++ /dev/null @@ -1,209 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1485340173501 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From 40550feb7837ef2891ce6f57e1772ba6e3b1db72 Mon Sep 17 00:00:00 2001 From: josephyearsley Date: Wed, 25 Jan 2017 12:48:59 +0000 Subject: [PATCH 3/9] Added PELU Activation --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 72364f9..bef4e83 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,7 @@ pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ +*.xml .coverage .coverage.* .cache From ed59a5fc0151451d7f58ef1a94b299af0dff114e Mon Sep 17 00:00:00 2001 From: josephyearsley Date: Wed, 25 Jan 2017 12:55:02 +0000 Subject: [PATCH 4/9] Removed Pycharms files --- .gitignore | 6 +++++- .idea/keras-contrib.iml | 12 ------------ 2 files changed, 5 insertions(+), 13 deletions(-) delete mode 100644 .idea/keras-contrib.iml diff --git a/.gitignore b/.gitignore index bef4e83..2c57e7e 100644 --- a/.gitignore +++ b/.gitignore @@ -34,10 +34,14 @@ var/ pip-log.txt pip-delete-this-directory.txt +# PyCharms +*.xml +*.iml +.idea/ + # Unit test / coverage reports htmlcov/ .tox/ -*.xml .coverage .coverage.* .cache diff --git a/.idea/keras-contrib.iml b/.idea/keras-contrib.iml deleted file mode 100644 index e98082a..0000000 --- a/.idea/keras-contrib.iml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - \ No newline at end of file From 63bd82a633afaeb85feb182d31354249fba12994 Mon Sep 17 00:00:00 2001 From: josephyearsley Date: Wed, 25 Jan 2017 22:28:24 +0000 Subject: [PATCH 5/9] Added Tests and Updated Doc String --- keras_contrib/layers/advanced_activations.py | 24 ++++++++++++------- pytest.ini | 10 ++++---- .../layers/test_advanced_activations.py | 14 +++++++++++ 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/keras_contrib/layers/advanced_activations.py b/keras_contrib/layers/advanced_activations.py index 199600b..2678f19 100644 --- a/keras_contrib/layers/advanced_activations.py +++ b/keras_contrib/layers/advanced_activations.py @@ -1,5 +1,6 @@ from .. import initializations from keras.engine import Layer +from keras.utils.generic_utils import get_custom_objects from .. import backend as K import numpy as np @@ -7,9 +8,9 @@ import numpy as np class PELU(Layer): """Parametric Exponential Linear Unit. It follows: - `f(x) = alphas * x for x < 0`, - `f(x) = x for x >= 0`, - where `alphas` is a learned array with the same shape as x. + `f(x) = alphas * (exp(x / betas) - 1) for x < 0`, + `f(x) = (alphas / betas) * x for x >= 0`, + where `alphas` & `betas` are learned arrays with the same shape as x. # Input shape Arbitrary. Use the keyword argument `input_shape` (tuple of integers, does not include the samples axis) @@ -17,7 +18,8 @@ class PELU(Layer): # Output shape Same shape as the input. # Arguments - init: initialization function for the weights. + alphas_init: initialization function for the alpha variable weights. + betas_init: initialization function for the beta variable weights. weights: initial weights, as a list of a single Numpy array. shared_axes: the axes along which to share learnable parameters for the activation function. @@ -31,9 +33,10 @@ class PELU(Layer): - [PARAMETRIC EXPONENTIAL LINEAR UNIT FOR DEEP CONVOLUTIONAL NEURAL NETWORKS](https://arxiv.org/abs/1605.09332v3) """ - def __init__(self, init='zero', weights=None, shared_axes=None, **kwargs): + def __init__(self, alphas_init='one', betas_init='one', weights=None, shared_axes=None, **kwargs): self.supports_masking = True - self.init = initializations.get(init) + self.alphas_init = initializations.get(alphas_init) + self.betas_init = initializations.get(betas_init) self.initial_weights = weights if not isinstance(shared_axes, (list, tuple)): self.shared_axes = [shared_axes] @@ -50,9 +53,9 @@ class PELU(Layer): self.param_broadcast[i - 1] = True # Initialised as ones to emulate the default ELU - self.alphas = K.ones(param_shape, dtype=K.floatx(), + self.alphas = self.alphas_init(param_shape, name='{}_alphas'.format(self.name)) - self.betas = K.ones(param_shape, dtype=K.floatx(), + self.betas = self.betas_init(param_shape, name='{}_betas'.format(self.name)) self.trainable_weights = [self.alphas, self.betas] @@ -71,6 +74,9 @@ class PELU(Layer): return pos + neg def get_config(self): - config = {'init': self.init.__name__} + config = {'alphas_init': self.alphas_init.__name__, + 'betas_init': self.betas_init.__name__} base_config = super(PELU, self).get_config() return dict(list(base_config.items()) + list(config.items())) + +get_custom_objects().update({"PELU": PELU}) diff --git a/pytest.ini b/pytest.ini index 079ed86..29e995d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,10 +1,10 @@ # Configuration of py.test [pytest] -addopts=-v - -n 2 - --durations=10 - --cov-report term-missing - --cov=keras +#addopts=-v +# -n 2 +# --durations=10 +# --cov-report term-missing +# --cov=keras # Do not run tests in the build folder norecursedirs= build diff --git a/tests/keras_contrib/layers/test_advanced_activations.py b/tests/keras_contrib/layers/test_advanced_activations.py index d510fc9..a235497 100644 --- a/tests/keras_contrib/layers/test_advanced_activations.py +++ b/tests/keras_contrib/layers/test_advanced_activations.py @@ -1,5 +1,19 @@ import pytest from keras.utils.test_utils import layer_test, keras_test +from keras_contrib import layers + +@keras_test +def test_pelu(): + from keras_contrib.layers.advanced_activations import PELU + layer_test(PELU, kwargs={}, + input_shape=(2, 3, 4)) + + +@keras_test +def test_pelu_share(): + from keras_contrib.layers.advanced_activations import PELU + layer_test(PELU, kwargs={'shared_axes': 1}, + input_shape=(2, 3, 4)) if __name__ == '__main__': From d84a9cc5bdb5cbd28f1cca20115413cb7a1c3e49 Mon Sep 17 00:00:00 2001 From: josephyearsley Date: Wed, 25 Jan 2017 22:47:34 +0000 Subject: [PATCH 6/9] Fixed PEP8 and Theano backend --- keras_contrib/layers/advanced_activations.py | 10 ++++++---- .../keras_contrib/layers/test_advanced_activations.py | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/keras_contrib/layers/advanced_activations.py b/keras_contrib/layers/advanced_activations.py index 2678f19..a9d709d 100644 --- a/keras_contrib/layers/advanced_activations.py +++ b/keras_contrib/layers/advanced_activations.py @@ -54,9 +54,9 @@ class PELU(Layer): # Initialised as ones to emulate the default ELU self.alphas = self.alphas_init(param_shape, - name='{}_alphas'.format(self.name)) + name='{}_alphas'.format(self.name)) self.betas = self.betas_init(param_shape, - name='{}_betas'.format(self.name)) + name='{}_betas'.format(self.name)) self.trainable_weights = [self.alphas, self.betas] @@ -65,11 +65,13 @@ class PELU(Layer): del self.initial_weights def call(self, x, mask=None): - pos = K.relu(x) * (self.alphas / self.betas) if K.backend() == 'theano': + pos = K.relu(x) * (K.pattern_broadcast(self.alphas, self.param_broadcast) / + K.pattern_broadcast(self.betas, self.param_broadcast)) neg = (K.pattern_broadcast(self.alphas, self.param_broadcast) * - (K.exp(x / self.betas) - 1)) + (K.exp(x / K.pattern_broadcast(self.betas, self.param_broadcast)) - 1)) else: + pos = K.relu(x) * (self.alphas / self.betas) neg = self.alphas * (K.exp(x / self.betas) - 1) return pos + neg diff --git a/tests/keras_contrib/layers/test_advanced_activations.py b/tests/keras_contrib/layers/test_advanced_activations.py index a235497..2254993 100644 --- a/tests/keras_contrib/layers/test_advanced_activations.py +++ b/tests/keras_contrib/layers/test_advanced_activations.py @@ -2,6 +2,7 @@ import pytest from keras.utils.test_utils import layer_test, keras_test from keras_contrib import layers + @keras_test def test_pelu(): from keras_contrib.layers.advanced_activations import PELU From 0163830c921cc9cff8cfb99d27ec50566ebdf9ed Mon Sep 17 00:00:00 2001 From: josephyearsley Date: Wed, 25 Jan 2017 23:26:55 +0000 Subject: [PATCH 7/9] Reset pytest.ini - my bad --- pytest.ini | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pytest.ini b/pytest.ini index 29e995d..079ed86 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,10 +1,10 @@ # Configuration of py.test [pytest] -#addopts=-v -# -n 2 -# --durations=10 -# --cov-report term-missing -# --cov=keras +addopts=-v + -n 2 + --durations=10 + --cov-report term-missing + --cov=keras # Do not run tests in the build folder norecursedirs= build From 1be552eb6de093f3f7c394b7f9edd5fc3e01d1dd Mon Sep 17 00:00:00 2001 From: Fariz Rahman Date: Thu, 26 Jan 2017 11:37:40 +0530 Subject: [PATCH 8/9] Import on top of file --- tests/keras_contrib/layers/test_advanced_activations.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/keras_contrib/layers/test_advanced_activations.py b/tests/keras_contrib/layers/test_advanced_activations.py index 2254993..0dbfb69 100644 --- a/tests/keras_contrib/layers/test_advanced_activations.py +++ b/tests/keras_contrib/layers/test_advanced_activations.py @@ -1,19 +1,17 @@ import pytest from keras.utils.test_utils import layer_test, keras_test -from keras_contrib import layers +from keras_contrib.layers import advanced_activations @keras_test def test_pelu(): - from keras_contrib.layers.advanced_activations import PELU - layer_test(PELU, kwargs={}, + layer_test(advanced_activations.PELU, kwargs={}, input_shape=(2, 3, 4)) @keras_test def test_pelu_share(): - from keras_contrib.layers.advanced_activations import PELU - layer_test(PELU, kwargs={'shared_axes': 1}, + layer_test(advanced_activations.PELU, kwargs={'shared_axes': 1}, input_shape=(2, 3, 4)) From 40ace94d89bb3e93bcf47cc2a60075adfbaec5bd Mon Sep 17 00:00:00 2001 From: Fariz Rahman Date: Thu, 26 Jan 2017 14:43:42 +0530 Subject: [PATCH 9/9] Bug fix --- keras_contrib/layers/advanced_activations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/keras_contrib/layers/advanced_activations.py b/keras_contrib/layers/advanced_activations.py index a9d709d..f83f044 100644 --- a/keras_contrib/layers/advanced_activations.py +++ b/keras_contrib/layers/advanced_activations.py @@ -66,14 +66,14 @@ class PELU(Layer): def call(self, x, mask=None): if K.backend() == 'theano': - pos = K.relu(x) * (K.pattern_broadcast(self.alphas, self.param_broadcast) / + pos = x * (K.pattern_broadcast(self.alphas, self.param_broadcast) / K.pattern_broadcast(self.betas, self.param_broadcast)) neg = (K.pattern_broadcast(self.alphas, self.param_broadcast) * (K.exp(x / K.pattern_broadcast(self.betas, self.param_broadcast)) - 1)) else: pos = K.relu(x) * (self.alphas / self.betas) neg = self.alphas * (K.exp(x / self.betas) - 1) - return pos + neg + return K.switch(x < 0., neg, pos) def get_config(self): config = {'alphas_init': self.alphas_init.__name__,