diff --git a/keras_contrib/backend/tensorflow_backend.py b/keras_contrib/backend/tensorflow_backend.py index 0f73034..ef43d06 100644 --- a/keras_contrib/backend/tensorflow_backend.py +++ b/keras_contrib/backend/tensorflow_backend.py @@ -58,7 +58,8 @@ def deconv3d(x, kernel, output_shape, strides=(1, 1, 1), raise ValueError('Unknown dim_ordering ' + str(dim_ordering)) x = _preprocess_conv3d_input(x, dim_ordering) - output_shape = _preprocess_deconv_output_shape(x, output_shape, dim_ordering) + output_shape = _preprocess_deconv_output_shape(x, output_shape, + dim_ordering) kernel = _preprocess_conv3d_kernel(kernel, dim_ordering) kernel = tf.transpose(kernel, (0, 1, 2, 4, 3)) padding = _preprocess_border_mode(border_mode) @@ -69,32 +70,35 @@ def deconv3d(x, kernel, output_shape, strides=(1, 1, 1), return _postprocess_conv3d_output(x, dim_ordering) -def extract_image_patches(X, ksizes, ssizes, border_mode="same", dim_ordering="tf"): +def extract_image_patches(x, ksizes, ssizes, border_mode="same", + dim_ordering="tf"): ''' Extract the patches from an image - Parameters - ---------- - X : The input image - ksizes : 2-d tuple with the kernel size - ssizes : 2-d tuple with the strides size - border_mode : 'same' or 'valid' - dim_ordering : 'tf' or 'th' - Returns - ------- - The (k_w,k_h) patches extracted - TF ==> (batch_size,w,h,k_w,k_h,c) - TH ==> (batch_size,w,h,c,k_w,k_h) + # Parameters + + x : The input image + ksizes : 2-d tuple with the kernel size + ssizes : 2-d tuple with the strides size + border_mode : 'same' or 'valid' + dim_ordering : 'tf' or 'th' + + # Returns + The (k_w,k_h) patches extracted + TF ==> (batch_size,w,h,k_w,k_h,c) + TH ==> (batch_size,w,h,c,k_w,k_h) ''' kernel = [1, ksizes[0], ksizes[1], 1] strides = [1, ssizes[0], ssizes[1], 1] padding = _preprocess_border_mode(border_mode) if dim_ordering == "th": - X = KTF.permute_dimensions(X, (0, 2, 3, 1)) - bs_i, w_i, h_i, ch_i = KTF.int_shape(X) - patches = tf.extract_image_patches(X, kernel, strides, [1, 1, 1, 1], padding) + x = KTF.permute_dimensions(x, (0, 2, 3, 1)) + bs_i, w_i, h_i, ch_i = KTF.int_shape(x) + patches = tf.extract_image_patches(x, kernel, strides, [1, 1, 1, 1], + padding) # Reshaping to fit Theano bs, w, h, ch = KTF.int_shape(patches) - patches = tf.reshape(tf.transpose(tf.reshape(patches, [bs, w, h, -1, ch_i]), [0, 1, 2, 4, 3]), + patches = tf.reshape(patches, [bs, w, h, -1, ch_i]) + patches = tf.reshape(tf.transpose(patches, [0, 1, 2, 4, 3]), [bs, w, h, ch_i, ksizes[0], ksizes[1]]) if dim_ordering == "tf": patches = KTF.permute_dimensions(patches, [0, 1, 2, 4, 5, 3]) diff --git a/keras_contrib/layers/convolutional.py b/keras_contrib/layers/convolutional.py index fe6c004..307b5e4 100644 --- a/keras_contrib/layers/convolutional.py +++ b/keras_contrib/layers/convolutional.py @@ -13,6 +13,7 @@ from keras.layers.convolutional import Convolution3D from keras.utils.generic_utils import get_custom_objects from keras.utils.np_utils import conv_output_length from keras.utils.np_utils import conv_input_length +import numpy as np class Deconvolution3D(Convolution3D): @@ -229,6 +230,237 @@ get_custom_objects().update({"Deconvolution3D": Deconvolution3D}) get_custom_objects().update({"Deconv3D": Deconv3D}) +class CosineConvolution2D(Layer): + """Cosine Normalized Convolution operator for filtering windows of two-dimensional inputs. + Cosine Normalization: Using Cosine Similarity Instead of Dot Product in Neural Networks + https://arxiv.org/pdf/1702.05870.pdf + + When using this layer as the first layer in a model, + provide the keyword argument `input_shape` + (tuple of integers, does not include the sample axis), + e.g. `input_shape=(3, 128, 128)` for 128x128 RGB pictures. + + # Examples + + ```python + # apply a 3x3 convolution with 64 output filters on a 256x256 image: + model = Sequential() + model.add(CosineConvolution2D(64, 3, 3, + border_mode='same', + input_shape=(3, 256, 256))) + # now model.output_shape == (None, 64, 256, 256) + + # add a 3x3 convolution on top, with 32 output filters: + model.add(CosineConvolution2D(32, 3, 3, border_mode='same')) + # now model.output_shape == (None, 32, 256, 256) + ``` + + # Arguments + nb_filter: Number of convolution filters to use. + nb_row: Number of rows in the convolution kernel. + nb_col: Number of columns in the convolution kernel. + init: name of initialization function for the weights of the layer + (see [initializations](../initializations.md)), or alternatively, + Theano function to use for weights initialization. + This parameter is only relevant if you don't pass + a `weights` argument. + activation: name of activation function to use + (see [activations](../activations.md)), + or alternatively, elementwise Theano function. + If you don't specify anything, no activation is applied + (ie. "linear" activation: a(x) = x). + weights: list of numpy arrays to set as initial weights. + border_mode: 'valid', 'same' or 'full' + ('full' requires the Theano backend). + subsample: tuple of length 2. Factor by which to subsample output. + Also called strides elsewhere. + W_regularizer: instance of [WeightRegularizer](../regularizers.md) + (eg. L1 or L2 regularization), applied to the main weights matrix. + b_regularizer: instance of [WeightRegularizer](../regularizers.md), + applied to the bias. + activity_regularizer: instance of [ActivityRegularizer](../regularizers.md), + applied to the network output. + W_constraint: instance of the [constraints](../constraints.md) module + (eg. maxnorm, nonneg), applied to the main weights matrix. + b_constraint: instance of the [constraints](../constraints.md) module, + applied to the bias. + dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension + (the depth) is at index 1, in 'tf' mode is it at index 3. + It defaults to the `image_dim_ordering` value found in your + Keras config file at `~/.keras/keras.json`. + If you never set it, then it will be "tf". + bias: whether to include a bias + (i.e. make the layer affine rather than linear). + + # Input shape + 4D tensor with shape: + `(samples, channels, rows, cols)` if dim_ordering='th' + or 4D tensor with shape: + `(samples, rows, cols, channels)` if dim_ordering='tf'. + + # Output shape + 4D tensor with shape: + `(samples, nb_filter, new_rows, new_cols)` if dim_ordering='th' + or 4D tensor with shape: + `(samples, new_rows, new_cols, nb_filter)` if dim_ordering='tf'. + `rows` and `cols` values might have changed due to padding. + """ + + def __init__(self, nb_filter, nb_row, nb_col, + init='glorot_uniform', activation=None, weights=None, + border_mode='valid', subsample=(1, 1), dim_ordering='default', + W_regularizer=None, b_regularizer=None, + activity_regularizer=None, + W_constraint=None, b_constraint=None, + bias=True, **kwargs): + if dim_ordering == 'default': + dim_ordering = K.image_dim_ordering() + if border_mode not in {'valid', 'same', 'full'}: + raise ValueError('Invalid border mode for CosineConvolution2D:', border_mode) + self.nb_filter = nb_filter + self.nb_row = nb_row + self.nb_col = nb_col + self.init = initializations.get(init) + self.activation = activations.get(activation) + self.border_mode = border_mode + self.subsample = tuple(subsample) + if dim_ordering not in {'tf', 'th'}: + raise ValueError('dim_ordering must be in {tf, th}.') + self.dim_ordering = dim_ordering + + self.W_regularizer = regularizers.get(W_regularizer) + self.b_regularizer = regularizers.get(b_regularizer) + self.activity_regularizer = regularizers.get(activity_regularizer) + + self.W_constraint = constraints.get(W_constraint) + self.b_constraint = constraints.get(b_constraint) + + self.bias = bias + self.input_spec = [InputSpec(ndim=4)] + self.initial_weights = weights + super(CosineConvolution2D, self).__init__(**kwargs) + + def build(self, input_shape): + if self.dim_ordering == 'th': + stack_size = input_shape[1] + self.W_shape = (self.nb_filter, stack_size, self.nb_row, self.nb_col) + self.W_norm_shape = (1, stack_size, self.nb_row, self.nb_col) + elif self.dim_ordering == 'tf': + stack_size = input_shape[3] + self.W_shape = (self.nb_row, self.nb_col, stack_size, self.nb_filter) + self.W_norm_shape = (self.nb_row, self.nb_col, stack_size, 1) + else: + raise ValueError('Invalid dim_ordering:', self.dim_ordering) + self.W = self.add_weight(self.W_shape, + initializer=functools.partial(self.init, + dim_ordering=self.dim_ordering), + name='{}_W'.format(self.name), + regularizer=self.W_regularizer, + constraint=self.W_constraint) + + self.W_norm = K.variable(np.ones(self.W_norm_shape), name='{}_W_norm'.format(self.name)) + + if self.bias: + self.b = self.add_weight((self.nb_filter,), + initializer='zero', + name='{}_b'.format(self.name), + regularizer=self.b_regularizer, + constraint=self.b_constraint) + else: + self.b = None + + if self.initial_weights is not None: + self.set_weights(self.initial_weights) + del self.initial_weights + self.built = True + + def get_output_shape_for(self, input_shape): + if self.dim_ordering == 'th': + rows = input_shape[2] + cols = input_shape[3] + elif self.dim_ordering == 'tf': + rows = input_shape[1] + cols = input_shape[2] + else: + raise ValueError('Invalid dim_ordering:', self.dim_ordering) + + rows = conv_output_length(rows, self.nb_row, + self.border_mode, self.subsample[0]) + cols = conv_output_length(cols, self.nb_col, + self.border_mode, self.subsample[1]) + + if self.dim_ordering == 'th': + return (input_shape[0], self.nb_filter, rows, cols) + elif self.dim_ordering == 'tf': + return (input_shape[0], rows, cols, self.nb_filter) + + def call(self, x, mask=None): + b, xb = 0., 0. + if self.dim_ordering == 'th': + W_sum_axes = [1, 2, 3] + if self.bias: + b = K.reshape(self.b, (self.nb_filter, 1, 1, 1)) + xb = 1. + elif self.dim_ordering == 'tf': + W_sum_axes = [0, 1, 2] + if self.bias: + b = K.reshape(self.b, (1, 1, 1, self.nb_filter)) + xb = 1. + + Wnorm = K.sqrt(K.sum(K.square(self.W), axis=W_sum_axes, keepdims=True) + K.square(b) + K.epsilon()) + xnorm = K.sqrt(K.conv2d(K.square(x), self.W_norm, strides=self.subsample, + border_mode=self.border_mode, + dim_ordering=self.dim_ordering, + filter_shape=self.W_norm_shape) + xb + K.epsilon()) + + W = self.W / Wnorm + + output = K.conv2d(x, W, strides=self.subsample, + border_mode=self.border_mode, + dim_ordering=self.dim_ordering, + filter_shape=self.W_shape) + + if K.backend() == 'theano': + xnorm = K.pattern_broadcast(xnorm, [False, True, False, False]) + + output /= xnorm + + if self.bias: + b /= Wnorm + if self.dim_ordering == 'th': + b = K.reshape(b, (1, self.nb_filter, 1, 1)) + elif self.dim_ordering == 'tf': + b = K.reshape(b, (1, 1, 1, self.nb_filter)) + else: + raise ValueError('Invalid dim_ordering:', self.dim_ordering) + b /= xnorm + output += b + output = self.activation(output) + return output + + def get_config(self): + config = {'nb_filter': self.nb_filter, + 'nb_row': self.nb_row, + 'nb_col': self.nb_col, + 'init': self.init.__name__, + 'activation': self.activation.__name__, + 'border_mode': self.border_mode, + 'subsample': self.subsample, + 'dim_ordering': self.dim_ordering, + 'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None, + 'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None, + 'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None, + 'W_constraint': self.W_constraint.get_config() if self.W_constraint else None, + 'b_constraint': self.b_constraint.get_config() if self.b_constraint else None, + 'bias': self.bias} + base_config = super(CosineConvolution2D, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + +CosineConv2D = CosineConvolution2D +get_custom_objects().update({"CosineConvolution2D": CosineConvolution2D}) +get_custom_objects().update({"CosineConv2D": CosineConv2D}) + + class SubPixelUpscaling(Layer): def __init__(self, scale_factor=2, dim_ordering='default', **kwargs): diff --git a/keras_contrib/layers/core.py b/keras_contrib/layers/core.py index 7e2b19c..4c5da2c 100644 --- a/keras_contrib/layers/core.py +++ b/keras_contrib/layers/core.py @@ -21,3 +21,163 @@ from keras.engine import Merge from keras.utils.generic_utils import func_dump from keras.utils.generic_utils import func_load from keras.utils.generic_utils import get_from_module +from keras.utils.generic_utils import get_custom_objects + + +class CosineDense(Layer): + """A cosine normalized densely-connected NN layer + Cosine Normalization: Using Cosine Similarity Instead of Dot Product in Neural Networks + https://arxiv.org/pdf/1702.05870.pdf + + # Example + + ```python + # as first layer in a sequential model: + model = Sequential() + model.add(CosineDense(32, input_dim=16)) + # now the model will take as input arrays of shape (*, 16) + # and output arrays of shape (*, 32) + + # this is equivalent to the above: + model = Sequential() + model.add(CosineDense(32, input_shape=(16,))) + + # after the first layer, you don't need to specify + # the size of the input anymore: + model.add(CosineDense(32)) + + **Note that a regular Dense layer may work better as the final layer + ``` + + # Arguments + output_dim: int > 0. + init: name of initialization function for the weights of the layer + (see [initializations](../initializations.md)), + or alternatively, Theano function to use for weights + initialization. This parameter is only relevant + if you don't pass a `weights` argument. + activation: name of activation function to use + (see [activations](../activations.md)), + or alternatively, elementwise Theano function. + If you don't specify anything, no activation is applied + (ie. "linear" activation: a(x) = x). + weights: list of Numpy arrays to set as initial weights. + The list should have 2 elements, of shape `(input_dim, output_dim)` + and (output_dim,) for weights and biases respectively. + W_regularizer: instance of [WeightRegularizer](../regularizers.md) + (eg. L1 or L2 regularization), applied to the main weights matrix. + b_regularizer: instance of [WeightRegularizer](../regularizers.md), + applied to the bias. + activity_regularizer: instance of [ActivityRegularizer](../regularizers.md), + applied to the network output. + W_constraint: instance of the [constraints](../constraints.md) module + (eg. maxnorm, nonneg), applied to the main weights matrix. + b_constraint: instance of the [constraints](../constraints.md) module, + applied to the bias. + bias: whether to include a bias + (i.e. make the layer affine rather than linear). + input_dim: dimensionality of the input (integer). This argument + (or alternatively, the keyword argument `input_shape`) + is required when using this layer as the first layer in a model. + + # Input shape + nD tensor with shape: `(nb_samples, ..., input_dim)`. + The most common situation would be + a 2D input with shape `(nb_samples, input_dim)`. + + # Output shape + nD tensor with shape: `(nb_samples, ..., output_dim)`. + For instance, for a 2D input with shape `(nb_samples, input_dim)`, + the output would have shape `(nb_samples, output_dim)`. + """ + + def __init__(self, output_dim, init='glorot_uniform', + activation=None, weights=None, + W_regularizer=None, b_regularizer=None, activity_regularizer=None, + W_constraint=None, b_constraint=None, + bias=True, input_dim=None, **kwargs): + self.init = initializations.get(init) + self.activation = activations.get(activation) + self.output_dim = output_dim + self.input_dim = input_dim + + self.W_regularizer = regularizers.get(W_regularizer) + self.b_regularizer = regularizers.get(b_regularizer) + self.activity_regularizer = regularizers.get(activity_regularizer) + + self.W_constraint = constraints.get(W_constraint) + self.b_constraint = constraints.get(b_constraint) + + self.bias = bias + self.initial_weights = weights + self.input_spec = [InputSpec(ndim='2+')] + + if self.input_dim: + kwargs['input_shape'] = (self.input_dim,) + super(CosineDense, self).__init__(**kwargs) + + def build(self, input_shape): + assert len(input_shape) >= 2 + input_dim = input_shape[-1] + self.input_dim = input_dim + self.input_spec = [InputSpec(dtype=K.floatx(), + ndim='2+')] + + self.W = self.add_weight((input_dim, self.output_dim), + initializer=self.init, + name='{}_W'.format(self.name), + regularizer=self.W_regularizer, + constraint=self.W_constraint) + if self.bias: + self.b = self.add_weight((self.output_dim,), + initializer='zero', + name='{}_b'.format(self.name), + regularizer=self.b_regularizer, + constraint=self.b_constraint) + else: + self.b = None + + if self.initial_weights is not None: + self.set_weights(self.initial_weights) + del self.initial_weights + self.built = True + + def call(self, x, mask=None): + if self.bias: + b, xb = self.b, 1. + else: + b, xb = 0., 0. + + xnorm = K.sqrt(K.sum(K.square(x), axis=-1, keepdims=True) + xb + K.epsilon()) + Wnorm = K.sqrt(K.sum(K.square(self.W), axis=0) + K.square(b) + K.epsilon()) + + xWnorm = (xnorm * Wnorm) + + output = K.dot(x, self.W) / xWnorm + if self.bias: + output += (self.b / xWnorm) + return self.activation(output) + + def get_output_shape_for(self, input_shape): + assert input_shape and len(input_shape) >= 2 + assert input_shape[-1] and input_shape[-1] == self.input_dim + output_shape = list(input_shape) + output_shape[-1] = self.output_dim + return tuple(output_shape) + + def get_config(self): + config = {'output_dim': self.output_dim, + 'init': self.init.__name__, + 'activation': self.activation.__name__, + 'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None, + 'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None, + 'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None, + 'W_constraint': self.W_constraint.get_config() if self.W_constraint else None, + 'b_constraint': self.b_constraint.get_config() if self.b_constraint else None, + 'bias': self.bias, + 'input_dim': self.input_dim} + base_config = super(CosineDense, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + +get_custom_objects().update({"CosineDense": CosineDense}) diff --git a/tests/keras_contrib/layers/test_convolutional.py b/tests/keras_contrib/layers/test_convolutional.py index 65ff23b..dcd1907 100644 --- a/tests/keras_contrib/layers/test_convolutional.py +++ b/tests/keras_contrib/layers/test_convolutional.py @@ -8,7 +8,7 @@ from keras.utils.np_utils import conv_input_length from keras import backend as K from keras_contrib import backend as KC from keras_contrib.layers import convolutional, pooling - +from keras.models import Sequential # TensorFlow does not support full convolution. if K.backend() == 'theano': @@ -77,6 +77,77 @@ def test_deconvolution_3d(): input_shape=(nb_samples, stack_size, kernel_dim1, kernel_dim2, kernel_dim3)) +@keras_test +def test_cosineconvolution_2d(): + nb_samples = 2 + nb_filter = 2 + stack_size = 3 + nb_row = 10 + nb_col = 6 + + if K.backend() == 'theano': + dim_ordering = 'th' + elif K.backend() == 'tensorflow': + dim_ordering = 'tf' + + for border_mode in _convolution_border_modes: + for subsample in [(1, 1), (2, 2)]: + for bias_mode in [True, False]: + if border_mode == 'same' and subsample != (1, 1): + continue + + layer_test(convolutional.CosineConvolution2D, + kwargs={'nb_filter': nb_filter, + 'nb_row': 3, + 'nb_col': 3, + 'border_mode': border_mode, + 'subsample': subsample, + 'bias': bias_mode, + 'dim_ordering': dim_ordering}, + input_shape=(nb_samples, nb_row, nb_col, stack_size)) + + layer_test(convolutional.CosineConvolution2D, + kwargs={'nb_filter': nb_filter, + 'nb_row': 3, + 'nb_col': 3, + 'border_mode': border_mode, + 'W_regularizer': 'l2', + 'b_regularizer': 'l2', + 'activity_regularizer': 'activity_l2', + 'subsample': subsample, + 'bias': bias_mode, + 'dim_ordering': dim_ordering}, + input_shape=(nb_samples, nb_row, nb_col, stack_size)) + + if dim_ordering == 'th': + X = np.random.randn(1, 3, 5, 5) + input_dim = (3, 5, 5) + W0 = X[:, :, ::-1, ::-1] + elif dim_ordering == 'tf': + X = np.random.randn(1, 5, 5, 3) + input_dim = (5, 5, 3) + W0 = X[0, :, :, :, None] + + model = Sequential() + model.add(convolutional.CosineConvolution2D(1, 5, 5, bias=True, input_shape=input_dim, dim_ordering=dim_ordering)) + model.compile(loss='mse', optimizer='rmsprop') + W = model.get_weights() + W[0] = W0 + W[1] = np.asarray([1.]) + model.set_weights(W) + out = model.predict(X) + assert_allclose(out, np.ones((1, 1, 1, 1), dtype=K.floatx()), atol=1e-5) + + model = Sequential() + model.add(convolutional.CosineConvolution2D(1, 5, 5, bias=False, input_shape=input_dim, dim_ordering=dim_ordering)) + model.compile(loss='mse', optimizer='rmsprop') + W = model.get_weights() + W[0] = -2 * W0 + model.set_weights(W) + out = model.predict(X) + assert_allclose(out, -np.ones((1, 1, 1, 1), dtype=K.floatx()), atol=1e-5) + + @keras_test def test_sub_pixel_upscaling(): nb_samples = 2 diff --git a/tests/keras_contrib/layers/test_core.py b/tests/keras_contrib/layers/test_core.py index fa517a4..7475323 100644 --- a/tests/keras_contrib/layers/test_core.py +++ b/tests/keras_contrib/layers/test_core.py @@ -5,6 +5,60 @@ from keras import backend as K from keras_contrib import backend as KC from keras_contrib.layers import core from keras.utils.test_utils import layer_test, keras_test +from numpy.testing import assert_allclose + + +@keras_test +def test_cosinedense(): + from keras import regularizers + from keras import constraints + from keras.models import Sequential + + layer_test(core.CosineDense, + kwargs={'output_dim': 3}, + input_shape=(3, 2)) + + layer_test(core.CosineDense, + kwargs={'output_dim': 3}, + input_shape=(3, 4, 2)) + + layer_test(core.CosineDense, + kwargs={'output_dim': 3}, + input_shape=(None, None, 2)) + + layer_test(core.CosineDense, + kwargs={'output_dim': 3}, + input_shape=(3, 4, 5, 2)) + + layer_test(core.CosineDense, + kwargs={'output_dim': 3, + 'W_regularizer': regularizers.l2(0.01), + 'b_regularizer': regularizers.l1(0.01), + 'activity_regularizer': regularizers.activity_l2(0.01), + 'W_constraint': constraints.MaxNorm(1), + 'b_constraint': constraints.MaxNorm(1)}, + input_shape=(3, 2)) + + X = np.random.randn(1, 20) + model = Sequential() + model.add(core.CosineDense(1, bias=True, input_shape=(20,))) + model.compile(loss='mse', optimizer='rmsprop') + W = model.get_weights() + W[0] = X.T + W[1] = np.asarray([1.]) + model.set_weights(W) + out = model.predict(X) + assert_allclose(out, np.ones((1, 1), dtype=K.floatx()), atol=1e-5) + + X = np.random.randn(1, 20) + model = Sequential() + model.add(core.CosineDense(1, bias=False, input_shape=(20,))) + model.compile(loss='mse', optimizer='rmsprop') + W = model.get_weights() + W[0] = -2 * X.T + model.set_weights(W) + out = model.predict(X) + assert_allclose(out, -np.ones((1, 1), dtype=K.floatx()), atol=1e-5) if __name__ == '__main__':