diff --git a/keras_contrib/layers/advanced_activations.py b/keras_contrib/layers/advanced_activations.py index 3ddaf25..60e3bb6 100644 --- a/keras_contrib/layers/advanced_activations.py +++ b/keras_contrib/layers/advanced_activations.py @@ -111,5 +111,122 @@ class PELU(Layer): base_config = super(PELU, self).get_config() return dict(list(base_config.items()) + list(config.items())) - get_custom_objects().update({'PELU': PELU}) + + +class SReLU(Layer): + """S-shaped Rectified Linear Unit. + + It follows: + `f(x) = t^r + a^r(x - t^r) for x >= t^r`, + `f(x) = x for t^r > x > t^l`, + `f(x) = t^l + a^l(x - t^l) for x <= t^l`. + + # Input shape + Arbitrary. Use the keyword argument `input_shape` + (tuple of integers, does not include the samples axis) + when using this layer as the first layer in a model. + + # Output shape + Same shape as the input. + + # Arguments + t_left_initializer: initializer function for the left part intercept + a_left_initializer: initializer function for the left part slope + t_right_initializer: initializer function for the right part intercept + a_right_initializer: initializer function for the right part slope + shared_axes: the axes along which to share learnable + parameters for the activation function. + For example, if the incoming feature maps + are from a 2D convolution + with output shape `(batch, height, width, channels)`, + and you wish to share parameters across space + so that each filter only has one set of parameters, + set `shared_axes=[1, 2]`. + + # References + - [Deep Learning with S-shaped Rectified Linear Activation Units](http://arxiv.org/abs/1512.07030) + """ + + def __init__(self, t_left_initializer='zeros', + a_left_initializer='glorot_uniform', + t_right_initializer='glorot_uniform', + a_right_initializer='ones', + shared_axes=None, + **kwargs): + self.supports_masking = True + self.t_left_initializer = initializers.get(t_left_initializer) + self.a_left_initializer = initializers.get(a_left_initializer) + self.t_right_initializer = initializers.get(t_right_initializer) + self.a_right_initializer = initializers.get(a_right_initializer) + if not isinstance(shared_axes, (list, tuple)): + self.shared_axes = [shared_axes] + else: + self.shared_axes = list(shared_axes) + super(SReLU, self).__init__(**kwargs) + + def build(self, input_shape): + param_shape = list(input_shape[1:]) + self.param_broadcast = [False] * len(param_shape) + if self.shared_axes is not None: + for i in self.shared_axes: + param_shape[i - 1] = 1 + self.param_broadcast[i - 1] = True + + self.t_left = self.add_weight(param_shape, + name='t_left', + initializer=self.t_left_initializer) + + self.a_left = self.add_weight(param_shape, + name='a_left', + initializer=self.a_left_initializer) + + self.t_right = self.add_weight(param_shape, + name='t_right', + initializer=self.t_right_initializer) + + self.a_right = self.add_weight(param_shape, + name='a_right', + initializer=self.a_right_initializer) + + # Set input spec + axes = {} + if self.shared_axes: + for i in range(1, len(input_shape)): + if i not in self.shared_axes: + axes[i] = input_shape[i] + self.input_spec = InputSpec(ndim=len(input_shape), axes=axes) + self.built = True + + def call(self, x, mask=None): + # ensure the the right part is always to the right of the left + t_right_actual = self.t_left + K.abs(self.t_right) + + if K.backend() == 'theano': + t_left = K.pattern_broadcast(self.t_left, self.param_broadcast) + a_left = K.pattern_broadcast(self.a_left, self.param_broadcast) + a_right = K.pattern_broadcast(self.a_right, self.param_broadcast) + t_right_actual = K.pattern_broadcast(t_right_actual, + self.param_broadcast) + else: + t_left = self.t_left + a_left = self.a_left + a_right = self.a_right + + y_left_and_center = t_left + K.relu(x - t_left, + a_left, + t_right_actual - t_left) + y_right = K.relu(x - t_right_actual) * a_right + return y_left_and_center + y_right + + def get_config(self): + config = { + 't_left_initializer': self.t_left_initializer, + 'a_left_initializer': self.a_left_initializer, + 't_right_initializer': self.t_right_initializer, + 'a_right_initializer': self.a_right_initializer + } + base_config = super(SReLU, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + +get_custom_objects().update({'SReLU': SReLU}) diff --git a/tests/keras_contrib/layers/test_advanced_activations.py b/tests/keras_contrib/layers/test_advanced_activations.py index 0dbfb69..f07a690 100644 --- a/tests/keras_contrib/layers/test_advanced_activations.py +++ b/tests/keras_contrib/layers/test_advanced_activations.py @@ -15,5 +15,16 @@ def test_pelu_share(): input_shape=(2, 3, 4)) +@keras_test +def test_srelu(): + layer_test(advanced_activations.SReLU, kwargs={}, + input_shape=(2, 3, 4)) + + +@keras_test +def test_srelu_share(): + layer_test(advanced_activations.SReLU, kwargs={'shared_axes': 1}, + input_shape=(2, 3, 4)) + if __name__ == '__main__': pytest.main([__file__])