From 6f9769339917dc6153afa8142aaf535b020c3fc2 Mon Sep 17 00:00:00 2001 From: josephyearsley Date: Wed, 25 Jan 2017 12:47:56 +0000 Subject: [PATCH] Added PELU Activation --- .idea/encodings.xml | 6 + .idea/keras-contrib.iml | 12 ++ .idea/misc.xml | 29 +++ .idea/modules.xml | 8 + .idea/vcs.xml | 6 + .idea/workspace.xml | 209 +++++++++++++++++++ keras_contrib/layers/advanced_activations.py | 72 +++++++ 7 files changed, 342 insertions(+) create mode 100644 .idea/encodings.xml create mode 100644 .idea/keras-contrib.iml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml create mode 100644 .idea/workspace.xml diff --git a/.idea/encodings.xml b/.idea/encodings.xml new file mode 100644 index 0000000..97626ba --- /dev/null +++ b/.idea/encodings.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/keras-contrib.iml b/.idea/keras-contrib.iml new file mode 100644 index 0000000..e98082a --- /dev/null +++ b/.idea/keras-contrib.iml @@ -0,0 +1,12 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..bc2d718 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..d4181d1 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..63feacb --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,209 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1485340173501 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/keras_contrib/layers/advanced_activations.py b/keras_contrib/layers/advanced_activations.py index 6b39244..199600b 100644 --- a/keras_contrib/layers/advanced_activations.py +++ b/keras_contrib/layers/advanced_activations.py @@ -2,3 +2,75 @@ from .. import initializations from keras.engine import Layer from .. import backend as K import numpy as np + + +class PELU(Layer): + """Parametric Exponential Linear Unit. + It follows: + `f(x) = alphas * x for x < 0`, + `f(x) = x for x >= 0`, + where `alphas` is a learned array with the same shape as x. + # Input shape + Arbitrary. Use the keyword argument `input_shape` + (tuple of integers, does not include the samples axis) + when using this layer as the first layer in a model. + # Output shape + Same shape as the input. + # Arguments + init: initialization function for the weights. + weights: initial weights, as a list of a single Numpy array. + shared_axes: the axes along which to share learnable + parameters for the activation function. + For example, if the incoming feature maps + are from a 2D convolution + with output shape `(batch, height, width, channels)`, + and you wish to share parameters across space + so that each filter only has one set of parameters, + set `shared_axes=[1, 2]`. + # References + - [PARAMETRIC EXPONENTIAL LINEAR UNIT FOR DEEP CONVOLUTIONAL NEURAL NETWORKS](https://arxiv.org/abs/1605.09332v3) + """ + + def __init__(self, init='zero', weights=None, shared_axes=None, **kwargs): + self.supports_masking = True + self.init = initializations.get(init) + self.initial_weights = weights + if not isinstance(shared_axes, (list, tuple)): + self.shared_axes = [shared_axes] + else: + self.shared_axes = list(shared_axes) + super(PELU, self).__init__(**kwargs) + + def build(self, input_shape): + param_shape = list(input_shape[1:]) + self.param_broadcast = [False] * len(param_shape) + if self.shared_axes[0] is not None: + for i in self.shared_axes: + param_shape[i - 1] = 1 + self.param_broadcast[i - 1] = True + + # Initialised as ones to emulate the default ELU + self.alphas = K.ones(param_shape, dtype=K.floatx(), + name='{}_alphas'.format(self.name)) + self.betas = K.ones(param_shape, dtype=K.floatx(), + name='{}_betas'.format(self.name)) + + self.trainable_weights = [self.alphas, self.betas] + + if self.initial_weights is not None: + self.set_weights(self.initial_weights) + del self.initial_weights + + def call(self, x, mask=None): + pos = K.relu(x) * (self.alphas / self.betas) + if K.backend() == 'theano': + neg = (K.pattern_broadcast(self.alphas, self.param_broadcast) * + (K.exp(x / self.betas) - 1)) + else: + neg = self.alphas * (K.exp(x / self.betas) - 1) + return pos + neg + + def get_config(self): + config = {'init': self.init.__name__} + base_config = super(PELU, self).get_config() + return dict(list(base_config.items()) + list(config.items()))