From 942d5c30c1c9bc6b08303b48c7192b0eff68c780 Mon Sep 17 00:00:00 2001 From: Valentin Date: Sat, 18 Nov 2017 16:54:06 +0100 Subject: [PATCH 1/3] adds swish activation function --- keras_contrib/layers/advanced_activations.py | 46 +++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/keras_contrib/layers/advanced_activations.py b/keras_contrib/layers/advanced_activations.py index 179856c..894166b 100644 --- a/keras_contrib/layers/advanced_activations.py +++ b/keras_contrib/layers/advanced_activations.py @@ -235,4 +235,48 @@ class SReLU(Layer): base_config = super(SReLU, self).get_config() return dict(list(base_config.items()) + list(config.items())) -get_custom_objects().update({'SReLU': SReLU}) +class Swish(Layer): + """ Swish (Ramachandranet al., 2017) + + # Input shape + Arbitrary. Use the keyword argument `input_shape` + (tuple of integers, does not include the samples axis) + when using this layer as the first layer in a model. + + # Output shape + Same shape as the input. + + # Arguments + beta: float >= 0. Scaling factor + if set to 1 and trainable set to False (default), Swish equals the SiLU activation (Elfwing et al., 2017) + trainable: whether to learn the scaling factor during training or not + + # References + - [Searching for Activation Functions](https://arxiv.org/abs/1710.05941) + - [Sigmoid-weighted linear units for neural network function approximation in reinforcement learning](https://arxiv.org/abs/1702.03118) + """ + + def __init__(self, beta=1.0, trainable=False, **kwargs): + super(Swish, self).__init__(**kwargs) + self.supports_masking = True + self.beta = beta + self.trainable = trainable + + def build(self, input_shape): + self.scaling_factor = K.variable(self.beta, + dtype=K.floatx(), + name='scaling_factor') + if self.trainable: + self._trainable_weights.append(self.scaling_factor) + super(Swish, self).build(input_shape) + + def call(self, inputs, mask=None): + return inputs * K.sigmoid(self.scaling_factor * inputs) + + def get_config(self): + config = {'beta': self.get_weights()[0] if self.trainable else self.beta, + 'trainable': self.trainable} + base_config = super(Swish, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + +get_custom_objects().update({'SReLU': SReLU, 'Swish': Swish}) From 2f95542184c74f734c93c933185f89b4ff724c30 Mon Sep 17 00:00:00 2001 From: Valentin Date: Sat, 18 Nov 2017 16:54:23 +0100 Subject: [PATCH 2/3] adds tests for swish activation --- .../layers/test_advanced_activations.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/keras_contrib/layers/test_advanced_activations.py b/tests/keras_contrib/layers/test_advanced_activations.py index f07a690..57e3d4a 100644 --- a/tests/keras_contrib/layers/test_advanced_activations.py +++ b/tests/keras_contrib/layers/test_advanced_activations.py @@ -26,5 +26,17 @@ def test_srelu_share(): layer_test(advanced_activations.SReLU, kwargs={'shared_axes': 1}, input_shape=(2, 3, 4)) +@keras_test +def test_swish_constant(): + layer_test(advanced_activations.Swish, kwargs={'beta': 1.0, 'trainable': False}, + input_shape=(2, 3, 4)) + + +@keras_test +def test_swish_trainable(): + layer_test(advanced_activations.Swish, kwargs={'beta': 1.0, 'trainable': True}, + input_shape=(2, 3, 4)) + + if __name__ == '__main__': pytest.main([__file__]) From 416bbe026cc0f4aae25a5b2062459c319179886f Mon Sep 17 00:00:00 2001 From: Valentin Date: Wed, 6 Dec 2017 21:36:06 +0100 Subject: [PATCH 3/3] add lines for pep8 --- keras_contrib/layers/advanced_activations.py | 5 ++++- tests/keras_contrib/layers/test_advanced_activations.py | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/keras_contrib/layers/advanced_activations.py b/keras_contrib/layers/advanced_activations.py index 894166b..7bf349e 100644 --- a/keras_contrib/layers/advanced_activations.py +++ b/keras_contrib/layers/advanced_activations.py @@ -235,6 +235,9 @@ class SReLU(Layer): base_config = super(SReLU, self).get_config() return dict(list(base_config.items()) + list(config.items())) +get_custom_objects().update({'SReLU': SReLU}) + + class Swish(Layer): """ Swish (Ramachandranet al., 2017) @@ -279,4 +282,4 @@ class Swish(Layer): base_config = super(Swish, self).get_config() return dict(list(base_config.items()) + list(config.items())) -get_custom_objects().update({'SReLU': SReLU, 'Swish': Swish}) +get_custom_objects().update({'Swish': Swish}) diff --git a/tests/keras_contrib/layers/test_advanced_activations.py b/tests/keras_contrib/layers/test_advanced_activations.py index 57e3d4a..8c71426 100644 --- a/tests/keras_contrib/layers/test_advanced_activations.py +++ b/tests/keras_contrib/layers/test_advanced_activations.py @@ -26,6 +26,7 @@ def test_srelu_share(): layer_test(advanced_activations.SReLU, kwargs={'shared_axes': 1}, input_shape=(2, 3, 4)) + @keras_test def test_swish_constant(): layer_test(advanced_activations.Swish, kwargs={'beta': 1.0, 'trainable': False},