From b10e9d795f1f75c720c076ecbc89df838b0b5a55 Mon Sep 17 00:00:00 2001
From: Michael Oliver <michael.d.oliver@gmail.com>
Date: Wed, 22 Feb 2017 11:05:37 -0800
Subject: [PATCH 01/23] Add cosine normalized dense

---
 keras_contrib/layers/core.py            | 153 ++++++++++++++++++++++++
 tests/keras_contrib/layers/test_core.py |  55 +++++++++
 2 files changed, 208 insertions(+)

diff --git a/keras_contrib/layers/core.py b/keras_contrib/layers/core.py
index 7e2b19c..2a66edf 100644
--- a/keras_contrib/layers/core.py
+++ b/keras_contrib/layers/core.py
@@ -21,3 +21,156 @@ from keras.engine import Merge
 from keras.utils.generic_utils import func_dump
 from keras.utils.generic_utils import func_load
 from keras.utils.generic_utils import get_from_module
+
+
+class CosineDense(Layer):
+    """A cosine normalized densely-connected NN layer
+    Cosine Normalization: Using Cosine Similarity Instead of Dot Product in Neural Networks
+    https://arxiv.org/pdf/1702.05870.pdf
+
+    # Example
+
+    ```python
+        # as first layer in a sequential model:
+        model = Sequential()
+        model.add(CosineDense(32, input_dim=16))
+        # now the model will take as input arrays of shape (*, 16)
+        # and output arrays of shape (*, 32)
+
+        # this is equivalent to the above:
+        model = Sequential()
+        model.add(CosineDense(32, input_shape=(16,)))
+
+        # after the first layer, you don't need to specify
+        # the size of the input anymore:
+        model.add(CosineDense(32))
+    ```
+
+    # Arguments
+        output_dim: int > 0.
+        init: name of initialization function for the weights of the layer
+            (see [initializations](../initializations.md)),
+            or alternatively, Theano function to use for weights
+            initialization. This parameter is only relevant
+            if you don't pass a `weights` argument.
+        activation: name of activation function to use
+            (see [activations](../activations.md)),
+            or alternatively, elementwise Theano function.
+            If you don't specify anything, no activation is applied
+            (ie. "linear" activation: a(x) = x).
+        weights: list of Numpy arrays to set as initial weights.
+            The list should have 2 elements, of shape `(input_dim, output_dim)`
+            and (output_dim,) for weights and biases respectively.
+        W_regularizer: instance of [WeightRegularizer](../regularizers.md)
+            (eg. L1 or L2 regularization), applied to the main weights matrix.
+        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
+            applied to the bias.
+        activity_regularizer: instance of [ActivityRegularizer](../regularizers.md),
+            applied to the network output.
+        W_constraint: instance of the [constraints](../constraints.md) module
+            (eg. maxnorm, nonneg), applied to the main weights matrix.
+        b_constraint: instance of the [constraints](../constraints.md) module,
+            applied to the bias.
+        bias: whether to include a bias
+            (i.e. make the layer affine rather than linear).
+        input_dim: dimensionality of the input (integer). This argument
+            (or alternatively, the keyword argument `input_shape`)
+            is required when using this layer as the first layer in a model.
+
+    # Input shape
+        nD tensor with shape: `(nb_samples, ..., input_dim)`.
+        The most common situation would be
+        a 2D input with shape `(nb_samples, input_dim)`.
+
+    # Output shape
+        nD tensor with shape: `(nb_samples, ..., output_dim)`.
+        For instance, for a 2D input with shape `(nb_samples, input_dim)`,
+        the output would have shape `(nb_samples, output_dim)`.
+    """
+
+    def __init__(self, output_dim, init='glorot_uniform',
+                 activation=None, weights=None,
+                 W_regularizer=None, b_regularizer=None, activity_regularizer=None,
+                 W_constraint=None, b_constraint=None,
+                 bias=True, input_dim=None, **kwargs):
+        self.init = initializations.get(init)
+        self.activation = activations.get(activation)
+        self.output_dim = output_dim
+        self.input_dim = input_dim
+
+        self.W_regularizer = regularizers.get(W_regularizer)
+        self.b_regularizer = regularizers.get(b_regularizer)
+        self.activity_regularizer = regularizers.get(activity_regularizer)
+
+        self.W_constraint = constraints.get(W_constraint)
+        self.b_constraint = constraints.get(b_constraint)
+
+        self.bias = bias
+        self.initial_weights = weights
+        self.input_spec = [InputSpec(ndim='2+')]
+
+        if self.input_dim:
+            kwargs['input_shape'] = (self.input_dim,)
+        super(CosineDense, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+        assert len(input_shape) >= 2
+        input_dim = input_shape[-1]
+        self.input_dim = input_dim
+        self.input_spec = [InputSpec(dtype=K.floatx(),
+                                     ndim='2+')]
+
+        self.W = self.add_weight((input_dim, self.output_dim),
+                                 initializer=self.init,
+                                 name='{}_W'.format(self.name),
+                                 regularizer=self.W_regularizer,
+                                 constraint=self.W_constraint)
+        if self.bias:
+            self.b = self.add_weight((self.output_dim,),
+                                     initializer='zero',
+                                     name='{}_b'.format(self.name),
+                                     regularizer=self.b_regularizer,
+                                     constraint=self.b_constraint)
+        else:
+            self.b = None
+
+        if self.initial_weights is not None:
+            self.set_weights(self.initial_weights)
+            del self.initial_weights
+        self.built = True
+
+    def call(self, x, mask=None):
+        if self.bias:
+            xnorm = K.sqrt(K.sum(K.square(x), axis=1, keepdims=True) + 1 + K.epsilon())
+            x /= xnorm
+            Wnorm = K.sqrt(K.sum(K.square(self.W), axis=0) + K.square(self.b) + K.epsilon())
+        else:
+            x /= K.sqrt(K.sum(K.square(x), axis=1, keepdims=True) + K.epsilon())
+            Wnorm = K.sqrt(K.sum(K.square(self.W), axis=0) + K.epsilon())
+
+        W = self.W / Wnorm
+        output = K.dot(x, W)
+        if self.bias:
+            output += (self.b / (xnorm*Wnorm))
+        return self.activation(output)
+
+    def get_output_shape_for(self, input_shape):
+        assert input_shape and len(input_shape) >= 2
+        assert input_shape[-1] and input_shape[-1] == self.input_dim
+        output_shape = list(input_shape)
+        output_shape[-1] = self.output_dim
+        return tuple(output_shape)
+
+    def get_config(self):
+        config = {'output_dim': self.output_dim,
+                  'init': self.init.__name__,
+                  'activation': self.activation.__name__,
+                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
+                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
+                  'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
+                  'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
+                  'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
+                  'bias': self.bias,
+                  'input_dim': self.input_dim}
+        base_config = super(CosineDense, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
diff --git a/tests/keras_contrib/layers/test_core.py b/tests/keras_contrib/layers/test_core.py
index fa517a4..a8e703d 100644
--- a/tests/keras_contrib/layers/test_core.py
+++ b/tests/keras_contrib/layers/test_core.py
@@ -5,6 +5,61 @@ from keras import backend as K
 from keras_contrib import backend as KC
 from keras_contrib.layers import core
 from keras.utils.test_utils import layer_test, keras_test
+from numpy.testing import assert_allclose
+
+
+@keras_test
+def test_cosinedense():
+    from keras import regularizers
+    from keras import constraints
+    from keras.models import Sequential
+
+    layer_test(core.CosineDense,
+               kwargs={'output_dim': 3},
+               input_shape=(3, 2))
+
+    layer_test(core.CosineDense,
+               kwargs={'output_dim': 3},
+               input_shape=(3, 4, 2))
+
+    layer_test(core.CosineDense,
+               kwargs={'output_dim': 3},
+               input_shape=(None, None, 2))
+
+    layer_test(core.CosineDense,
+               kwargs={'output_dim': 3},
+               input_shape=(3, 4, 5, 2))
+
+    layer_test(core.CosineDense,
+               kwargs={'output_dim': 3,
+                       'W_regularizer': regularizers.l2(0.01),
+                       'b_regularizer': regularizers.l1(0.01),
+                       'activity_regularizer': regularizers.activity_l2(0.01),
+                       'W_constraint': constraints.MaxNorm(1),
+                       'b_constraint': constraints.MaxNorm(1)},
+               input_shape=(3, 2))
+
+    X = np.random.randn(1, 20)
+    model = Sequential()
+    model.add(core.CosineDense(1, bias=True, input_shape=(20,)))
+    model.compile(loss='mse', optimizer='rmsprop')
+    W = model.get_weights()
+    W[0] = X.T
+    W[1] = np.asarray([1.])
+    model.set_weights(W)
+    out = model.predict(X)
+    assert_allclose(out, np.ones((1, 1), dtype=K.floatx()))
+
+    X = np.random.randn(1, 20)
+    model = Sequential()
+    model.add(core.CosineDense(1, bias=False, input_shape=(20,)))
+    model.compile(loss='mse', optimizer='rmsprop')
+    W = model.get_weights()
+    W[0] = -X.T
+    model.set_weights(W)
+    out = model.predict(X)
+    assert_allclose(out, -np.ones((1, 1), dtype=K.floatx()))
+
 
 
 if __name__ == '__main__':

From 82b92142cf1975e7f0d653223a1b4bed395d0f65 Mon Sep 17 00:00:00 2001
From: Michael Oliver <michael.d.oliver@gmail.com>
Date: Wed, 22 Feb 2017 11:26:36 -0800
Subject: [PATCH 02/23] fixes and doc updates

---
 keras_contrib/layers/core.py            | 4 +++-
 tests/keras_contrib/layers/test_core.py | 1 -
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/keras_contrib/layers/core.py b/keras_contrib/layers/core.py
index 2a66edf..4048a90 100644
--- a/keras_contrib/layers/core.py
+++ b/keras_contrib/layers/core.py
@@ -44,6 +44,8 @@ class CosineDense(Layer):
         # after the first layer, you don't need to specify
         # the size of the input anymore:
         model.add(CosineDense(32))
+
+        **Note that a regular Dense layer may work better as the final layer
     ```
 
     # Arguments
@@ -151,7 +153,7 @@ class CosineDense(Layer):
         W = self.W / Wnorm
         output = K.dot(x, W)
         if self.bias:
-            output += (self.b / (xnorm*Wnorm))
+            output += (self.b / (xnorm * Wnorm))
         return self.activation(output)
 
     def get_output_shape_for(self, input_shape):
diff --git a/tests/keras_contrib/layers/test_core.py b/tests/keras_contrib/layers/test_core.py
index a8e703d..63c6734 100644
--- a/tests/keras_contrib/layers/test_core.py
+++ b/tests/keras_contrib/layers/test_core.py
@@ -61,6 +61,5 @@ def test_cosinedense():
     assert_allclose(out, -np.ones((1, 1), dtype=K.floatx()))
 
 
-
 if __name__ == '__main__':
     pytest.main([__file__])

From ac44e1a19cb3fccb769dbc22b5cf93a0f29228e3 Mon Sep 17 00:00:00 2001
From: Michael Oliver <michael.d.oliver@gmail.com>
Date: Wed, 22 Feb 2017 11:33:43 -0800
Subject: [PATCH 03/23] fix serialization

---
 keras_contrib/layers/core.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/keras_contrib/layers/core.py b/keras_contrib/layers/core.py
index 4048a90..47de531 100644
--- a/keras_contrib/layers/core.py
+++ b/keras_contrib/layers/core.py
@@ -21,6 +21,7 @@ from keras.engine import Merge
 from keras.utils.generic_utils import func_dump
 from keras.utils.generic_utils import func_load
 from keras.utils.generic_utils import get_from_module
+from keras.utils.generic_utils import get_custom_objects
 
 
 class CosineDense(Layer):
@@ -176,3 +177,6 @@ class CosineDense(Layer):
                   'input_dim': self.input_dim}
         base_config = super(CosineDense, self).get_config()
         return dict(list(base_config.items()) + list(config.items()))
+
+
+get_custom_objects().update({"CosineDense": CosineDense})

From 000927f7c95776d90ff281a7edab7034a8a46251 Mon Sep 17 00:00:00 2001
From: Michael Oliver <michael.d.oliver@gmail.com>
Date: Wed, 22 Feb 2017 11:45:42 -0800
Subject: [PATCH 04/23] fix >2D case

---
 keras_contrib/layers/core.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/keras_contrib/layers/core.py b/keras_contrib/layers/core.py
index 47de531..bc142a5 100644
--- a/keras_contrib/layers/core.py
+++ b/keras_contrib/layers/core.py
@@ -144,11 +144,11 @@ class CosineDense(Layer):
 
     def call(self, x, mask=None):
         if self.bias:
-            xnorm = K.sqrt(K.sum(K.square(x), axis=1, keepdims=True) + 1 + K.epsilon())
+            xnorm = K.sqrt(K.sum(K.square(x), axis=-1, keepdims=True) + 1 + K.epsilon())
             x /= xnorm
             Wnorm = K.sqrt(K.sum(K.square(self.W), axis=0) + K.square(self.b) + K.epsilon())
         else:
-            x /= K.sqrt(K.sum(K.square(x), axis=1, keepdims=True) + K.epsilon())
+            x /= K.sqrt(K.sum(K.square(x), axis=-1, keepdims=True) + K.epsilon())
             Wnorm = K.sqrt(K.sum(K.square(self.W), axis=0) + K.epsilon())
 
         W = self.W / Wnorm

From 538b9cd45ac33700a00b87e703255a573e9503d8 Mon Sep 17 00:00:00 2001
From: Michael Oliver <michael.d.oliver@gmail.com>
Date: Wed, 22 Feb 2017 11:59:38 -0800
Subject: [PATCH 05/23] change atol on allclose

---
 tests/keras_contrib/layers/test_core.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/keras_contrib/layers/test_core.py b/tests/keras_contrib/layers/test_core.py
index 63c6734..64f4b4d 100644
--- a/tests/keras_contrib/layers/test_core.py
+++ b/tests/keras_contrib/layers/test_core.py
@@ -48,7 +48,7 @@ def test_cosinedense():
     W[1] = np.asarray([1.])
     model.set_weights(W)
     out = model.predict(X)
-    assert_allclose(out, np.ones((1, 1), dtype=K.floatx()))
+    assert_allclose(out, np.ones((1, 1), dtype=K.floatx()), atol=1e-5)
 
     X = np.random.randn(1, 20)
     model = Sequential()
@@ -58,7 +58,7 @@ def test_cosinedense():
     W[0] = -X.T
     model.set_weights(W)
     out = model.predict(X)
-    assert_allclose(out, -np.ones((1, 1), dtype=K.floatx()))
+    assert_allclose(out, -np.ones((1, 1), dtype=K.floatx()), atol=1e-5)
 
 
 if __name__ == '__main__':

From f996579d4c4321e2af81c848b8aa127530eaec1d Mon Sep 17 00:00:00 2001
From: Michael Oliver <michael.d.oliver@gmail.com>
Date: Wed, 22 Feb 2017 13:06:52 -0800
Subject: [PATCH 06/23] initial commit

---
 keras_contrib/layers/convolutional.py | 222 ++++++++++++++++++++++++++
 1 file changed, 222 insertions(+)

diff --git a/keras_contrib/layers/convolutional.py b/keras_contrib/layers/convolutional.py
index faf1d03..2bf4659 100644
--- a/keras_contrib/layers/convolutional.py
+++ b/keras_contrib/layers/convolutional.py
@@ -13,6 +13,7 @@ from keras.layers.convolutional import Convolution3D
 from keras.utils.generic_utils import get_custom_objects
 from keras.utils.np_utils import conv_output_length
 from keras.utils.np_utils import conv_input_length
+import numpy as np
 
 
 class Deconvolution3D(Convolution3D):
@@ -227,3 +228,224 @@ class Deconvolution3D(Convolution3D):
 Deconv3D = Deconvolution3D
 get_custom_objects().update({"Deconvolution3D": Deconvolution3D})
 get_custom_objects().update({"Deconv3D": Deconv3D})
+
+
+class CosineConvolution2D(Layer):
+    """Convolution operator for filtering windows of two-dimensional inputs.
+
+    When using this layer as the first layer in a model,
+    provide the keyword argument `input_shape`
+    (tuple of integers, does not include the sample axis),
+    e.g. `input_shape=(3, 128, 128)` for 128x128 RGB pictures.
+
+    # Examples
+
+    ```python
+        # apply a 3x3 convolution with 64 output filters on a 256x256 image:
+        model = Sequential()
+        model.add(Convolution2D(64, 3, 3,
+                                border_mode='same',
+                                input_shape=(3, 256, 256)))
+        # now model.output_shape == (None, 64, 256, 256)
+
+        # add a 3x3 convolution on top, with 32 output filters:
+        model.add(Convolution2D(32, 3, 3, border_mode='same'))
+        # now model.output_shape == (None, 32, 256, 256)
+    ```
+
+    # Arguments
+        nb_filter: Number of convolution filters to use.
+        nb_row: Number of rows in the convolution kernel.
+        nb_col: Number of columns in the convolution kernel.
+        init: name of initialization function for the weights of the layer
+            (see [initializations](../initializations.md)), or alternatively,
+            Theano function to use for weights initialization.
+            This parameter is only relevant if you don't pass
+            a `weights` argument.
+        activation: name of activation function to use
+            (see [activations](../activations.md)),
+            or alternatively, elementwise Theano function.
+            If you don't specify anything, no activation is applied
+            (ie. "linear" activation: a(x) = x).
+        weights: list of numpy arrays to set as initial weights.
+        border_mode: 'valid', 'same' or 'full'
+            ('full' requires the Theano backend).
+        subsample: tuple of length 2. Factor by which to subsample output.
+            Also called strides elsewhere.
+        W_regularizer: instance of [WeightRegularizer](../regularizers.md)
+            (eg. L1 or L2 regularization), applied to the main weights matrix.
+        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
+            applied to the bias.
+        activity_regularizer: instance of [ActivityRegularizer](../regularizers.md),
+            applied to the network output.
+        W_constraint: instance of the [constraints](../constraints.md) module
+            (eg. maxnorm, nonneg), applied to the main weights matrix.
+        b_constraint: instance of the [constraints](../constraints.md) module,
+            applied to the bias.
+        dim_ordering: 'th' or 'tf'. In 'th' mode, the channels dimension
+            (the depth) is at index 1, in 'tf' mode is it at index 3.
+            It defaults to the `image_dim_ordering` value found in your
+            Keras config file at `~/.keras/keras.json`.
+            If you never set it, then it will be "tf".
+        bias: whether to include a bias
+            (i.e. make the layer affine rather than linear).
+
+    # Input shape
+        4D tensor with shape:
+        `(samples, channels, rows, cols)` if dim_ordering='th'
+        or 4D tensor with shape:
+        `(samples, rows, cols, channels)` if dim_ordering='tf'.
+
+    # Output shape
+        4D tensor with shape:
+        `(samples, nb_filter, new_rows, new_cols)` if dim_ordering='th'
+        or 4D tensor with shape:
+        `(samples, new_rows, new_cols, nb_filter)` if dim_ordering='tf'.
+        `rows` and `cols` values might have changed due to padding.
+    """
+
+    def __init__(self, nb_filter, nb_row, nb_col,
+                 init='glorot_uniform', activation=None, weights=None,
+                 border_mode='valid', subsample=(1, 1), dim_ordering='default',
+                 W_regularizer=None, b_regularizer=None,
+                 activity_regularizer=None,
+                 W_constraint=None, b_constraint=None,
+                 bias=True, **kwargs):
+        if dim_ordering == 'default':
+            dim_ordering = K.image_dim_ordering()
+        if border_mode not in {'valid', 'same', 'full'}:
+            raise ValueError('Invalid border mode for Convolution2D:', border_mode)
+        self.nb_filter = nb_filter
+        self.nb_row = nb_row
+        self.nb_col = nb_col
+        self.init = initializations.get(init)
+        self.activation = activations.get(activation)
+        self.border_mode = border_mode
+        self.subsample = tuple(subsample)
+        if dim_ordering not in {'tf', 'th'}:
+            raise ValueError('dim_ordering must be in {tf, th}.')
+        self.dim_ordering = dim_ordering
+
+        self.W_regularizer = regularizers.get(W_regularizer)
+        self.b_regularizer = regularizers.get(b_regularizer)
+        self.activity_regularizer = regularizers.get(activity_regularizer)
+
+        self.W_constraint = constraints.get(W_constraint)
+        self.b_constraint = constraints.get(b_constraint)
+
+        self.bias = bias
+        self.input_spec = [InputSpec(ndim=4)]
+        self.initial_weights = weights
+        super(CosineConvolution2D, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+        if self.dim_ordering == 'th':
+            stack_size = input_shape[1]
+            self.W_shape = (self.nb_filter, stack_size, self.nb_row, self.nb_col)
+            self.W_norm_shape = (1, stack_size, self.nb_row, self.nb_col)
+        elif self.dim_ordering == 'tf':
+            stack_size = input_shape[3]
+            self.W_shape = (self.nb_row, self.nb_col, stack_size, self.nb_filter)
+            self.W_norm_shape = (self.nb_row, self.nb_col, stack_size, 1)
+        else:
+            raise ValueError('Invalid dim_ordering:', self.dim_ordering)
+        self.W = self.add_weight(self.W_shape,
+                                 initializer=functools.partial(self.init,
+                                                               dim_ordering=self.dim_ordering),
+                                 name='{}_W'.format(self.name),
+                                 regularizer=self.W_regularizer,
+                                 constraint=self.W_constraint)
+
+        self.W_norm = K.variable(np.ones(self.W_norm_shape), name='{}_W_norm'.format(self.name))
+
+        if self.bias:
+            self.b = self.add_weight((self.nb_filter,),
+                                     initializer='zero',
+                                     name='{}_b'.format(self.name),
+                                     regularizer=self.b_regularizer,
+                                     constraint=self.b_constraint)
+        else:
+            self.b = None
+
+        if self.initial_weights is not None:
+            self.set_weights(self.initial_weights)
+            del self.initial_weights
+        self.built = True
+
+    def get_output_shape_for(self, input_shape):
+        if self.dim_ordering == 'th':
+            rows = input_shape[2]
+            cols = input_shape[3]
+        elif self.dim_ordering == 'tf':
+            rows = input_shape[1]
+            cols = input_shape[2]
+        else:
+            raise ValueError('Invalid dim_ordering:', self.dim_ordering)
+
+        rows = conv_output_length(rows, self.nb_row,
+                                  self.border_mode, self.subsample[0])
+        cols = conv_output_length(cols, self.nb_col,
+                                  self.border_mode, self.subsample[1])
+
+        if self.dim_ordering == 'th':
+            return (input_shape[0], self.nb_filter, rows, cols)
+        elif self.dim_ordering == 'tf':
+            return (input_shape[0], rows, cols, self.nb_filter)
+
+    def call(self, x, mask=None):
+        if self.dim_ordering == 'th':
+            W_sum_axes = [1, 2, 3]
+            b = K.reshape(self.b, (self.nb_filter, 1, 1, 1))
+        elif self.dim_ordering == 'tf':
+            W_sum_axes = [0, 1, 2]
+            b = K.reshape(self.b, (1, 1, 1, self.nb_filter))
+
+        Wnorm = K.sqrt(K.sum(K.square(self.W), axis=W_sum_axes, keepdims=True) + K.square(b) + K.epsilon())
+        W = self.W / Wnorm
+
+        output = K.conv2d(x, W, strides=self.subsample,
+                          border_mode=self.border_mode,
+                          dim_ordering=self.dim_ordering,
+                          filter_shape=self.W_shape)
+
+        xnorm = K.sqrt(K.conv2d(x**2, self.W_norm, strides=self.subsample,
+                  border_mode=self.border_mode,
+                  dim_ordering=self.dim_ordering,
+                  filter_shape=self.W_shape) + K.epsilon())
+
+        if K.backend() == 'theano':
+            xnorm = K.pattern_broadcast(xnorm, [False, True, False, False])
+
+        output /= xnorm
+
+        if self.bias:
+            if self.dim_ordering == 'th':
+                output += K.reshape(self.b, (1, self.nb_filter, 1, 1))
+            elif self.dim_ordering == 'tf':
+                output += K.reshape(self.b, (1, 1, 1, self.nb_filter))
+            else:
+                raise ValueError('Invalid dim_ordering:', self.dim_ordering)
+        output = self.activation(output)
+        return output
+
+    def get_config(self):
+        config = {'nb_filter': self.nb_filter,
+                  'nb_row': self.nb_row,
+                  'nb_col': self.nb_col,
+                  'init': self.init.__name__,
+                  'activation': self.activation.__name__,
+                  'border_mode': self.border_mode,
+                  'subsample': self.subsample,
+                  'dim_ordering': self.dim_ordering,
+                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
+                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
+                  'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
+                  'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
+                  'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
+                  'bias': self.bias}
+        base_config = super(CosineConvolution2D, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+CosineConv2D = CosineConvolution2D
+get_custom_objects().update({"CosineConvolution2D": CosineConvolution2D})
+get_custom_objects().update({"CosineConv2D": CosineConv2D})
\ No newline at end of file

From 4dc4821ff7af568cc3f9e3f4f74f512fa9db7778 Mon Sep 17 00:00:00 2001
From: Michael Oliver <michael.d.oliver@gmail.com>
Date: Wed, 22 Feb 2017 14:46:23 -0800
Subject: [PATCH 07/23] Cosine Normalized Dense Layer (#36)

* Add cosine normalized dense

* fixes and doc updates

* fix serialization

* fix >2D case

* change atol on allclose
---
 keras_contrib/layers/core.py            | 159 ++++++++++++++++++++++++
 tests/keras_contrib/layers/test_core.py |  54 ++++++++
 2 files changed, 213 insertions(+)

diff --git a/keras_contrib/layers/core.py b/keras_contrib/layers/core.py
index 7e2b19c..bc142a5 100644
--- a/keras_contrib/layers/core.py
+++ b/keras_contrib/layers/core.py
@@ -21,3 +21,162 @@ from keras.engine import Merge
 from keras.utils.generic_utils import func_dump
 from keras.utils.generic_utils import func_load
 from keras.utils.generic_utils import get_from_module
+from keras.utils.generic_utils import get_custom_objects
+
+
+class CosineDense(Layer):
+    """A cosine normalized densely-connected NN layer
+    Cosine Normalization: Using Cosine Similarity Instead of Dot Product in Neural Networks
+    https://arxiv.org/pdf/1702.05870.pdf
+
+    # Example
+
+    ```python
+        # as first layer in a sequential model:
+        model = Sequential()
+        model.add(CosineDense(32, input_dim=16))
+        # now the model will take as input arrays of shape (*, 16)
+        # and output arrays of shape (*, 32)
+
+        # this is equivalent to the above:
+        model = Sequential()
+        model.add(CosineDense(32, input_shape=(16,)))
+
+        # after the first layer, you don't need to specify
+        # the size of the input anymore:
+        model.add(CosineDense(32))
+
+        **Note that a regular Dense layer may work better as the final layer
+    ```
+
+    # Arguments
+        output_dim: int > 0.
+        init: name of initialization function for the weights of the layer
+            (see [initializations](../initializations.md)),
+            or alternatively, Theano function to use for weights
+            initialization. This parameter is only relevant
+            if you don't pass a `weights` argument.
+        activation: name of activation function to use
+            (see [activations](../activations.md)),
+            or alternatively, elementwise Theano function.
+            If you don't specify anything, no activation is applied
+            (ie. "linear" activation: a(x) = x).
+        weights: list of Numpy arrays to set as initial weights.
+            The list should have 2 elements, of shape `(input_dim, output_dim)`
+            and (output_dim,) for weights and biases respectively.
+        W_regularizer: instance of [WeightRegularizer](../regularizers.md)
+            (eg. L1 or L2 regularization), applied to the main weights matrix.
+        b_regularizer: instance of [WeightRegularizer](../regularizers.md),
+            applied to the bias.
+        activity_regularizer: instance of [ActivityRegularizer](../regularizers.md),
+            applied to the network output.
+        W_constraint: instance of the [constraints](../constraints.md) module
+            (eg. maxnorm, nonneg), applied to the main weights matrix.
+        b_constraint: instance of the [constraints](../constraints.md) module,
+            applied to the bias.
+        bias: whether to include a bias
+            (i.e. make the layer affine rather than linear).
+        input_dim: dimensionality of the input (integer). This argument
+            (or alternatively, the keyword argument `input_shape`)
+            is required when using this layer as the first layer in a model.
+
+    # Input shape
+        nD tensor with shape: `(nb_samples, ..., input_dim)`.
+        The most common situation would be
+        a 2D input with shape `(nb_samples, input_dim)`.
+
+    # Output shape
+        nD tensor with shape: `(nb_samples, ..., output_dim)`.
+        For instance, for a 2D input with shape `(nb_samples, input_dim)`,
+        the output would have shape `(nb_samples, output_dim)`.
+    """
+
+    def __init__(self, output_dim, init='glorot_uniform',
+                 activation=None, weights=None,
+                 W_regularizer=None, b_regularizer=None, activity_regularizer=None,
+                 W_constraint=None, b_constraint=None,
+                 bias=True, input_dim=None, **kwargs):
+        self.init = initializations.get(init)
+        self.activation = activations.get(activation)
+        self.output_dim = output_dim
+        self.input_dim = input_dim
+
+        self.W_regularizer = regularizers.get(W_regularizer)
+        self.b_regularizer = regularizers.get(b_regularizer)
+        self.activity_regularizer = regularizers.get(activity_regularizer)
+
+        self.W_constraint = constraints.get(W_constraint)
+        self.b_constraint = constraints.get(b_constraint)
+
+        self.bias = bias
+        self.initial_weights = weights
+        self.input_spec = [InputSpec(ndim='2+')]
+
+        if self.input_dim:
+            kwargs['input_shape'] = (self.input_dim,)
+        super(CosineDense, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+        assert len(input_shape) >= 2
+        input_dim = input_shape[-1]
+        self.input_dim = input_dim
+        self.input_spec = [InputSpec(dtype=K.floatx(),
+                                     ndim='2+')]
+
+        self.W = self.add_weight((input_dim, self.output_dim),
+                                 initializer=self.init,
+                                 name='{}_W'.format(self.name),
+                                 regularizer=self.W_regularizer,
+                                 constraint=self.W_constraint)
+        if self.bias:
+            self.b = self.add_weight((self.output_dim,),
+                                     initializer='zero',
+                                     name='{}_b'.format(self.name),
+                                     regularizer=self.b_regularizer,
+                                     constraint=self.b_constraint)
+        else:
+            self.b = None
+
+        if self.initial_weights is not None:
+            self.set_weights(self.initial_weights)
+            del self.initial_weights
+        self.built = True
+
+    def call(self, x, mask=None):
+        if self.bias:
+            xnorm = K.sqrt(K.sum(K.square(x), axis=-1, keepdims=True) + 1 + K.epsilon())
+            x /= xnorm
+            Wnorm = K.sqrt(K.sum(K.square(self.W), axis=0) + K.square(self.b) + K.epsilon())
+        else:
+            x /= K.sqrt(K.sum(K.square(x), axis=-1, keepdims=True) + K.epsilon())
+            Wnorm = K.sqrt(K.sum(K.square(self.W), axis=0) + K.epsilon())
+
+        W = self.W / Wnorm
+        output = K.dot(x, W)
+        if self.bias:
+            output += (self.b / (xnorm * Wnorm))
+        return self.activation(output)
+
+    def get_output_shape_for(self, input_shape):
+        assert input_shape and len(input_shape) >= 2
+        assert input_shape[-1] and input_shape[-1] == self.input_dim
+        output_shape = list(input_shape)
+        output_shape[-1] = self.output_dim
+        return tuple(output_shape)
+
+    def get_config(self):
+        config = {'output_dim': self.output_dim,
+                  'init': self.init.__name__,
+                  'activation': self.activation.__name__,
+                  'W_regularizer': self.W_regularizer.get_config() if self.W_regularizer else None,
+                  'b_regularizer': self.b_regularizer.get_config() if self.b_regularizer else None,
+                  'activity_regularizer': self.activity_regularizer.get_config() if self.activity_regularizer else None,
+                  'W_constraint': self.W_constraint.get_config() if self.W_constraint else None,
+                  'b_constraint': self.b_constraint.get_config() if self.b_constraint else None,
+                  'bias': self.bias,
+                  'input_dim': self.input_dim}
+        base_config = super(CosineDense, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+
+get_custom_objects().update({"CosineDense": CosineDense})
diff --git a/tests/keras_contrib/layers/test_core.py b/tests/keras_contrib/layers/test_core.py
index fa517a4..64f4b4d 100644
--- a/tests/keras_contrib/layers/test_core.py
+++ b/tests/keras_contrib/layers/test_core.py
@@ -5,6 +5,60 @@ from keras import backend as K
 from keras_contrib import backend as KC
 from keras_contrib.layers import core
 from keras.utils.test_utils import layer_test, keras_test
+from numpy.testing import assert_allclose
+
+
+@keras_test
+def test_cosinedense():
+    from keras import regularizers
+    from keras import constraints
+    from keras.models import Sequential
+
+    layer_test(core.CosineDense,
+               kwargs={'output_dim': 3},
+               input_shape=(3, 2))
+
+    layer_test(core.CosineDense,
+               kwargs={'output_dim': 3},
+               input_shape=(3, 4, 2))
+
+    layer_test(core.CosineDense,
+               kwargs={'output_dim': 3},
+               input_shape=(None, None, 2))
+
+    layer_test(core.CosineDense,
+               kwargs={'output_dim': 3},
+               input_shape=(3, 4, 5, 2))
+
+    layer_test(core.CosineDense,
+               kwargs={'output_dim': 3,
+                       'W_regularizer': regularizers.l2(0.01),
+                       'b_regularizer': regularizers.l1(0.01),
+                       'activity_regularizer': regularizers.activity_l2(0.01),
+                       'W_constraint': constraints.MaxNorm(1),
+                       'b_constraint': constraints.MaxNorm(1)},
+               input_shape=(3, 2))
+
+    X = np.random.randn(1, 20)
+    model = Sequential()
+    model.add(core.CosineDense(1, bias=True, input_shape=(20,)))
+    model.compile(loss='mse', optimizer='rmsprop')
+    W = model.get_weights()
+    W[0] = X.T
+    W[1] = np.asarray([1.])
+    model.set_weights(W)
+    out = model.predict(X)
+    assert_allclose(out, np.ones((1, 1), dtype=K.floatx()), atol=1e-5)
+
+    X = np.random.randn(1, 20)
+    model = Sequential()
+    model.add(core.CosineDense(1, bias=False, input_shape=(20,)))
+    model.compile(loss='mse', optimizer='rmsprop')
+    W = model.get_weights()
+    W[0] = -X.T
+    model.set_weights(W)
+    out = model.predict(X)
+    assert_allclose(out, -np.ones((1, 1), dtype=K.floatx()), atol=1e-5)
 
 
 if __name__ == '__main__':

From 8141f9ee2204c28d104f9f7df229e4bf2f866e06 Mon Sep 17 00:00:00 2001
From: Michael Oliver <michael.d.oliver@gmail.com>
Date: Wed, 22 Feb 2017 15:04:54 -0800
Subject: [PATCH 08/23] Update to test for scale invariance

---
 tests/keras_contrib/layers/test_core.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/keras_contrib/layers/test_core.py b/tests/keras_contrib/layers/test_core.py
index 64f4b4d..6adf010 100644
--- a/tests/keras_contrib/layers/test_core.py
+++ b/tests/keras_contrib/layers/test_core.py
@@ -55,7 +55,7 @@ def test_cosinedense():
     model.add(core.CosineDense(1, bias=False, input_shape=(20,)))
     model.compile(loss='mse', optimizer='rmsprop')
     W = model.get_weights()
-    W[0] = -X.T
+    W[0] = -2*X.T
     model.set_weights(W)
     out = model.predict(X)
     assert_allclose(out, -np.ones((1, 1), dtype=K.floatx()), atol=1e-5)

From 0e417813da62337be284cef936da832d2c1b9493 Mon Sep 17 00:00:00 2001
From: Michael Oliver <michael.d.oliver@gmail.com>
Date: Wed, 22 Feb 2017 16:30:17 -0800
Subject: [PATCH 09/23] finish cosine conv2d

---
 keras_contrib/layers/convolutional.py         | 36 ++++++----
 .../layers/test_convolutional.py              | 70 ++++++++++++++++++-
 tests/keras_contrib/layers/test_core.py       |  2 +-
 3 files changed, 93 insertions(+), 15 deletions(-)

diff --git a/keras_contrib/layers/convolutional.py b/keras_contrib/layers/convolutional.py
index 2bf4659..a20d6a7 100644
--- a/keras_contrib/layers/convolutional.py
+++ b/keras_contrib/layers/convolutional.py
@@ -231,7 +231,9 @@ get_custom_objects().update({"Deconv3D": Deconv3D})
 
 
 class CosineConvolution2D(Layer):
-    """Convolution operator for filtering windows of two-dimensional inputs.
+    """Cosine Normalized Convolution operator for filtering windows of two-dimensional inputs.
+    Cosine Normalization: Using Cosine Similarity Instead of Dot Product in Neural Networks
+    https://arxiv.org/pdf/1702.05870.pdf
 
     When using this layer as the first layer in a model,
     provide the keyword argument `input_shape`
@@ -243,13 +245,13 @@ class CosineConvolution2D(Layer):
     ```python
         # apply a 3x3 convolution with 64 output filters on a 256x256 image:
         model = Sequential()
-        model.add(Convolution2D(64, 3, 3,
+        model.add(CosineConvolution2D(64, 3, 3,
                                 border_mode='same',
                                 input_shape=(3, 256, 256)))
         # now model.output_shape == (None, 64, 256, 256)
 
         # add a 3x3 convolution on top, with 32 output filters:
-        model.add(Convolution2D(32, 3, 3, border_mode='same'))
+        model.add(CosineConvolution2D(32, 3, 3, border_mode='same'))
         # now model.output_shape == (None, 32, 256, 256)
     ```
 
@@ -314,7 +316,7 @@ class CosineConvolution2D(Layer):
         if dim_ordering == 'default':
             dim_ordering = K.image_dim_ordering()
         if border_mode not in {'valid', 'same', 'full'}:
-            raise ValueError('Invalid border mode for Convolution2D:', border_mode)
+            raise ValueError('Invalid border mode for CosineConvolution2D:', border_mode)
         self.nb_filter = nb_filter
         self.nb_row = nb_row
         self.nb_col = nb_col
@@ -393,14 +395,24 @@ class CosineConvolution2D(Layer):
             return (input_shape[0], rows, cols, self.nb_filter)
 
     def call(self, x, mask=None):
+        b, xb = 0, 0
         if self.dim_ordering == 'th':
             W_sum_axes = [1, 2, 3]
-            b = K.reshape(self.b, (self.nb_filter, 1, 1, 1))
+            if self.bias:
+                b = K.reshape(self.b, (self.nb_filter, 1, 1, 1))
+                xb = 1
         elif self.dim_ordering == 'tf':
             W_sum_axes = [0, 1, 2]
-            b = K.reshape(self.b, (1, 1, 1, self.nb_filter))
+            if self.bias:
+                b = K.reshape(self.b, (1, 1, 1, self.nb_filter))
+                xb = 1
 
         Wnorm = K.sqrt(K.sum(K.square(self.W), axis=W_sum_axes, keepdims=True) + K.square(b) + K.epsilon())
+        xnorm = K.sqrt(K.conv2d(x**2, self.W_norm, strides=self.subsample,
+                       border_mode=self.border_mode,
+                       dim_ordering=self.dim_ordering,
+                       filter_shape=self.W_shape) + xb + K.epsilon())
+
         W = self.W / Wnorm
 
         output = K.conv2d(x, W, strides=self.subsample,
@@ -408,23 +420,21 @@ class CosineConvolution2D(Layer):
                           dim_ordering=self.dim_ordering,
                           filter_shape=self.W_shape)
 
-        xnorm = K.sqrt(K.conv2d(x**2, self.W_norm, strides=self.subsample,
-                  border_mode=self.border_mode,
-                  dim_ordering=self.dim_ordering,
-                  filter_shape=self.W_shape) + K.epsilon())
-
         if K.backend() == 'theano':
             xnorm = K.pattern_broadcast(xnorm, [False, True, False, False])
 
         output /= xnorm
 
         if self.bias:
+            b /= Wnorm
             if self.dim_ordering == 'th':
-                output += K.reshape(self.b, (1, self.nb_filter, 1, 1))
+                b = K.reshape(b, (1, self.nb_filter, 1, 1))
             elif self.dim_ordering == 'tf':
-                output += K.reshape(self.b, (1, 1, 1, self.nb_filter))
+                b = K.reshape(b, (1, 1, 1, self.nb_filter))
             else:
                 raise ValueError('Invalid dim_ordering:', self.dim_ordering)
+            b /= xnorm
+            output += b
         output = self.activation(output)
         return output
 
diff --git a/tests/keras_contrib/layers/test_convolutional.py b/tests/keras_contrib/layers/test_convolutional.py
index 7b11fc7..6ec2d2a 100644
--- a/tests/keras_contrib/layers/test_convolutional.py
+++ b/tests/keras_contrib/layers/test_convolutional.py
@@ -7,7 +7,7 @@ from keras.utils.np_utils import conv_input_length
 from keras import backend as K
 from keras_contrib import backend as KC
 from keras_contrib.layers import convolutional, pooling
-
+from keras.models import Sequential
 
 # TensorFlow does not support full convolution.
 if K.backend() == 'theano':
@@ -75,6 +75,74 @@ def test_deconvolution_3d():
                                    'subsample': subsample},
                            input_shape=(nb_samples, stack_size, kernel_dim1, kernel_dim2, kernel_dim3))
 
+@keras_test
+def test_cosineconvolution_2d():
+    nb_samples = 2
+    nb_filter = 2
+    stack_size = 3
+    nb_row = 10
+    nb_col = 6
+
+    for border_mode in _convolution_border_modes:
+        for subsample in [(1, 1), (2, 2)]:
+          for bias_mode in [True, False]:
+            if border_mode == 'same' and subsample != (1, 1):
+                continue
+
+            layer_test(convolutional.CosineConvolution2D,
+                       kwargs={'nb_filter': nb_filter,
+                               'nb_row': 3,
+                               'nb_col': 3,
+                               'border_mode': border_mode,
+                               'subsample': subsample,
+                               'bias': bias_mode},
+                       input_shape=(nb_samples, nb_row, nb_col, stack_size))
+
+            layer_test(convolutional.CosineConvolution2D,
+                       kwargs={'nb_filter': nb_filter,
+                               'nb_row': 3,
+                               'nb_col': 3,
+                               'border_mode': border_mode,
+                               'W_regularizer': 'l2',
+                               'b_regularizer': 'l2',
+                               'activity_regularizer': 'activity_l2',
+                               'subsample': subsample,
+                               'bias': bias_mode},
+                       input_shape=(nb_samples, nb_row, nb_col, stack_size))
+
+    dim_ordering = K.image_dim_ordering()
+    assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
+
+    if dim_ordering == 'th':
+        X = np.random.randn(1, 3, 5, 5)
+        input_dim = (3, 5, 5)
+        W0 = X[:, :, ::-1, ::-1]
+    elif dim_ordering == 'tf':
+        X = np.random.randn(1, 5, 5, 3)
+        input_dim = (5, 5, 3)
+        W0 = X[:, ::-1, ::-1, :]
+
+    model = Sequential()
+    model.add(convolutional.CosineConvolution2D(1, 5, 5, bias=True, input_shape=input_dim))
+    model.compile(loss='mse', optimizer='rmsprop')
+    W = model.get_weights()
+    W[0] = W0
+    W[1] = np.asarray([1.])
+    model.set_weights(W)
+    out = model.predict(X)
+    assert_allclose(out, np.ones((1, 1, 1, 1), dtype=K.floatx()), atol=1e-5)
+
+    model = Sequential()
+    model.add(convolutional.CosineConvolution2D(1, 5, 5, bias=False, input_shape=input_dim))
+    model.compile(loss='mse', optimizer='rmsprop')
+    W = model.get_weights()
+    W[0] = -2*W0
+    model.set_weights(W)
+    out = model.predict(X)
+    assert_allclose(out, -np.ones((1, 1, 1, 1), dtype=K.floatx()), atol=1e-5)
+
+
+
 
 if __name__ == '__main__':
     pytest.main([__file__])
diff --git a/tests/keras_contrib/layers/test_core.py b/tests/keras_contrib/layers/test_core.py
index 64f4b4d..6adf010 100644
--- a/tests/keras_contrib/layers/test_core.py
+++ b/tests/keras_contrib/layers/test_core.py
@@ -55,7 +55,7 @@ def test_cosinedense():
     model.add(core.CosineDense(1, bias=False, input_shape=(20,)))
     model.compile(loss='mse', optimizer='rmsprop')
     W = model.get_weights()
-    W[0] = -X.T
+    W[0] = -2*X.T
     model.set_weights(W)
     out = model.predict(X)
     assert_allclose(out, -np.ones((1, 1), dtype=K.floatx()), atol=1e-5)

From 2b0092da2ea1d87dee26de1b317dd2a9622be1e8 Mon Sep 17 00:00:00 2001
From: Michael Oliver <michael.d.oliver@gmail.com>
Date: Wed, 22 Feb 2017 16:33:35 -0800
Subject: [PATCH 10/23] Update test_core.py

Make PEP8
---
 tests/keras_contrib/layers/test_core.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/keras_contrib/layers/test_core.py b/tests/keras_contrib/layers/test_core.py
index 6adf010..7475323 100644
--- a/tests/keras_contrib/layers/test_core.py
+++ b/tests/keras_contrib/layers/test_core.py
@@ -55,7 +55,7 @@ def test_cosinedense():
     model.add(core.CosineDense(1, bias=False, input_shape=(20,)))
     model.compile(loss='mse', optimizer='rmsprop')
     W = model.get_weights()
-    W[0] = -2*X.T
+    W[0] = -2 * X.T
     model.set_weights(W)
     out = model.predict(X)
     assert_allclose(out, -np.ones((1, 1), dtype=K.floatx()), atol=1e-5)

From 7576c6498511ab564b4120a943b6ff19569205b2 Mon Sep 17 00:00:00 2001
From: Michael Oliver <michael.d.oliver@gmail.com>
Date: Wed, 22 Feb 2017 16:34:15 -0800
Subject: [PATCH 11/23] fix pep8 test

---
 tests/keras_contrib/layers/test_core.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/keras_contrib/layers/test_core.py b/tests/keras_contrib/layers/test_core.py
index 6adf010..7475323 100644
--- a/tests/keras_contrib/layers/test_core.py
+++ b/tests/keras_contrib/layers/test_core.py
@@ -55,7 +55,7 @@ def test_cosinedense():
     model.add(core.CosineDense(1, bias=False, input_shape=(20,)))
     model.compile(loss='mse', optimizer='rmsprop')
     W = model.get_weights()
-    W[0] = -2*X.T
+    W[0] = -2 * X.T
     model.set_weights(W)
     out = model.predict(X)
     assert_allclose(out, -np.ones((1, 1), dtype=K.floatx()), atol=1e-5)

From 18600c2c6757cab4c7606107769cd99f22557e39 Mon Sep 17 00:00:00 2001
From: Michael Oliver <michael.d.oliver@gmail.com>
Date: Wed, 22 Feb 2017 16:34:46 -0800
Subject: [PATCH 12/23] fix pep8 test

---
 tests/keras_contrib/layers/test_convolutional.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/keras_contrib/layers/test_convolutional.py b/tests/keras_contrib/layers/test_convolutional.py
index 6ec2d2a..042a094 100644
--- a/tests/keras_contrib/layers/test_convolutional.py
+++ b/tests/keras_contrib/layers/test_convolutional.py
@@ -136,7 +136,7 @@ def test_cosineconvolution_2d():
     model.add(convolutional.CosineConvolution2D(1, 5, 5, bias=False, input_shape=input_dim))
     model.compile(loss='mse', optimizer='rmsprop')
     W = model.get_weights()
-    W[0] = -2*W0
+    W[0] = -2 * W0
     model.set_weights(W)
     out = model.predict(X)
     assert_allclose(out, -np.ones((1, 1, 1, 1), dtype=K.floatx()), atol=1e-5)

From 36c0c9e600c9942aeb72b63ea66d827aff1cab26 Mon Sep 17 00:00:00 2001
From: Michael Oliver <michael.d.oliver@gmail.com>
Date: Wed, 22 Feb 2017 16:47:38 -0800
Subject: [PATCH 13/23] PEP8 fixes

---
 keras_contrib/layers/convolutional.py         |  2 +-
 .../layers/test_convolutional.py              | 47 +++++++++----------
 2 files changed, 24 insertions(+), 25 deletions(-)

diff --git a/keras_contrib/layers/convolutional.py b/keras_contrib/layers/convolutional.py
index a20d6a7..8495d38 100644
--- a/keras_contrib/layers/convolutional.py
+++ b/keras_contrib/layers/convolutional.py
@@ -458,4 +458,4 @@ class CosineConvolution2D(Layer):
 
 CosineConv2D = CosineConvolution2D
 get_custom_objects().update({"CosineConvolution2D": CosineConvolution2D})
-get_custom_objects().update({"CosineConv2D": CosineConv2D})
\ No newline at end of file
+get_custom_objects().update({"CosineConv2D": CosineConv2D})
diff --git a/tests/keras_contrib/layers/test_convolutional.py b/tests/keras_contrib/layers/test_convolutional.py
index 042a094..483d5cb 100644
--- a/tests/keras_contrib/layers/test_convolutional.py
+++ b/tests/keras_contrib/layers/test_convolutional.py
@@ -75,6 +75,7 @@ def test_deconvolution_3d():
                                    'subsample': subsample},
                            input_shape=(nb_samples, stack_size, kernel_dim1, kernel_dim2, kernel_dim3))
 
+
 @keras_test
 def test_cosineconvolution_2d():
     nb_samples = 2
@@ -85,30 +86,30 @@ def test_cosineconvolution_2d():
 
     for border_mode in _convolution_border_modes:
         for subsample in [(1, 1), (2, 2)]:
-          for bias_mode in [True, False]:
-            if border_mode == 'same' and subsample != (1, 1):
-                continue
+            for bias_mode in [True, False]:
+                if border_mode == 'same' and subsample != (1, 1):
+                    continue
 
-            layer_test(convolutional.CosineConvolution2D,
-                       kwargs={'nb_filter': nb_filter,
-                               'nb_row': 3,
-                               'nb_col': 3,
-                               'border_mode': border_mode,
-                               'subsample': subsample,
-                               'bias': bias_mode},
-                       input_shape=(nb_samples, nb_row, nb_col, stack_size))
+                layer_test(convolutional.CosineConvolution2D,
+                           kwargs={'nb_filter': nb_filter,
+                                   'nb_row': 3,
+                                   'nb_col': 3,
+                                   'border_mode': border_mode,
+                                   'subsample': subsample,
+                                   'bias': bias_mode},
+                           input_shape=(nb_samples, nb_row, nb_col, stack_size))
 
-            layer_test(convolutional.CosineConvolution2D,
-                       kwargs={'nb_filter': nb_filter,
-                               'nb_row': 3,
-                               'nb_col': 3,
-                               'border_mode': border_mode,
-                               'W_regularizer': 'l2',
-                               'b_regularizer': 'l2',
-                               'activity_regularizer': 'activity_l2',
-                               'subsample': subsample,
-                               'bias': bias_mode},
-                       input_shape=(nb_samples, nb_row, nb_col, stack_size))
+                layer_test(convolutional.CosineConvolution2D,
+                           kwargs={'nb_filter': nb_filter,
+                                   'nb_row': 3,
+                                   'nb_col': 3,
+                                   'border_mode': border_mode,
+                                   'W_regularizer': 'l2',
+                                   'b_regularizer': 'l2',
+                                   'activity_regularizer': 'activity_l2',
+                                   'subsample': subsample,
+                                   'bias': bias_mode},
+                           input_shape=(nb_samples, nb_row, nb_col, stack_size))
 
     dim_ordering = K.image_dim_ordering()
     assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
@@ -142,7 +143,5 @@ def test_cosineconvolution_2d():
     assert_allclose(out, -np.ones((1, 1, 1, 1), dtype=K.floatx()), atol=1e-5)
 
 
-
-
 if __name__ == '__main__':
     pytest.main([__file__])

From 5b273df0369e2a1218ef9c498f5e57fa608115bc Mon Sep 17 00:00:00 2001
From: Michael Oliver <michael.d.oliver@gmail.com>
Date: Wed, 22 Feb 2017 18:08:22 -0800
Subject: [PATCH 14/23] filter shape bug fix

---
 keras_contrib/layers/convolutional.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/keras_contrib/layers/convolutional.py b/keras_contrib/layers/convolutional.py
index 8495d38..088ce72 100644
--- a/keras_contrib/layers/convolutional.py
+++ b/keras_contrib/layers/convolutional.py
@@ -395,23 +395,23 @@ class CosineConvolution2D(Layer):
             return (input_shape[0], rows, cols, self.nb_filter)
 
     def call(self, x, mask=None):
-        b, xb = 0, 0
+        b, xb = 0., 0.
         if self.dim_ordering == 'th':
             W_sum_axes = [1, 2, 3]
             if self.bias:
                 b = K.reshape(self.b, (self.nb_filter, 1, 1, 1))
-                xb = 1
+                xb = 1.
         elif self.dim_ordering == 'tf':
             W_sum_axes = [0, 1, 2]
             if self.bias:
                 b = K.reshape(self.b, (1, 1, 1, self.nb_filter))
-                xb = 1
+                xb = 1.
 
         Wnorm = K.sqrt(K.sum(K.square(self.W), axis=W_sum_axes, keepdims=True) + K.square(b) + K.epsilon())
-        xnorm = K.sqrt(K.conv2d(x**2, self.W_norm, strides=self.subsample,
+        xnorm = K.sqrt(K.conv2d(K.square(x), self.W_norm, strides=self.subsample,
                        border_mode=self.border_mode,
                        dim_ordering=self.dim_ordering,
-                       filter_shape=self.W_shape) + xb + K.epsilon())
+                       filter_shape=self.W_norm_shape) + xb + K.epsilon())
 
         W = self.W / Wnorm
 

From d8ca85f445c7fc20d5d64cab7da402b78f817f0d Mon Sep 17 00:00:00 2001
From: Michael Oliver <michael.d.oliver@gmail.com>
Date: Wed, 22 Feb 2017 18:23:39 -0800
Subject: [PATCH 15/23] fix tf dim ordering test

---
 tests/keras_contrib/layers/test_convolutional.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/keras_contrib/layers/test_convolutional.py b/tests/keras_contrib/layers/test_convolutional.py
index 483d5cb..1d4bd68 100644
--- a/tests/keras_contrib/layers/test_convolutional.py
+++ b/tests/keras_contrib/layers/test_convolutional.py
@@ -121,7 +121,7 @@ def test_cosineconvolution_2d():
     elif dim_ordering == 'tf':
         X = np.random.randn(1, 5, 5, 3)
         input_dim = (5, 5, 3)
-        W0 = X[:, ::-1, ::-1, :]
+        W0 = X[0, ::-1, ::-1, :, None]
 
     model = Sequential()
     model.add(convolutional.CosineConvolution2D(1, 5, 5, bias=True, input_shape=input_dim))

From 03155630c9c5b5a5d0d4fe085d30dd74687023ab Mon Sep 17 00:00:00 2001
From: Michael Oliver <michael.d.oliver@gmail.com>
Date: Wed, 22 Feb 2017 18:30:39 -0800
Subject: [PATCH 16/23] remove filter reversal for tf test

---
 tests/keras_contrib/layers/test_convolutional.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/keras_contrib/layers/test_convolutional.py b/tests/keras_contrib/layers/test_convolutional.py
index 1d4bd68..79cda33 100644
--- a/tests/keras_contrib/layers/test_convolutional.py
+++ b/tests/keras_contrib/layers/test_convolutional.py
@@ -121,7 +121,7 @@ def test_cosineconvolution_2d():
     elif dim_ordering == 'tf':
         X = np.random.randn(1, 5, 5, 3)
         input_dim = (5, 5, 3)
-        W0 = X[0, ::-1, ::-1, :, None]
+        W0 = X[0, :, :, :, None]
 
     model = Sequential()
     model.add(convolutional.CosineConvolution2D(1, 5, 5, bias=True, input_shape=input_dim))

From 65db762520b90041f2276f8eb70fb461550bf0ac Mon Sep 17 00:00:00 2001
From: Michael Oliver <michael.d.oliver@gmail.com>
Date: Wed, 22 Feb 2017 20:37:42 -0800
Subject: [PATCH 17/23] make dim ordering match backend for test

---
 tests/keras_contrib/layers/test_convolutional.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/tests/keras_contrib/layers/test_convolutional.py b/tests/keras_contrib/layers/test_convolutional.py
index 79cda33..1120a7c 100644
--- a/tests/keras_contrib/layers/test_convolutional.py
+++ b/tests/keras_contrib/layers/test_convolutional.py
@@ -84,6 +84,11 @@ def test_cosineconvolution_2d():
     nb_row = 10
     nb_col = 6
 
+    if K.backend() == 'theano':
+        dim_ordering = 'th'
+    elif K.backend() == 'tensorflow':
+        dim_ordering = 'tf'
+
     for border_mode in _convolution_border_modes:
         for subsample in [(1, 1), (2, 2)]:
             for bias_mode in [True, False]:
@@ -96,7 +101,8 @@ def test_cosineconvolution_2d():
                                    'nb_col': 3,
                                    'border_mode': border_mode,
                                    'subsample': subsample,
-                                   'bias': bias_mode},
+                                   'bias': bias_mode,
+                                   'dim_ordering': dim_ordering},
                            input_shape=(nb_samples, nb_row, nb_col, stack_size))
 
                 layer_test(convolutional.CosineConvolution2D,
@@ -108,7 +114,8 @@ def test_cosineconvolution_2d():
                                    'b_regularizer': 'l2',
                                    'activity_regularizer': 'activity_l2',
                                    'subsample': subsample,
-                                   'bias': bias_mode},
+                                   'bias': bias_mode,
+                                   'dim_ordering': dim_ordering},
                            input_shape=(nb_samples, nb_row, nb_col, stack_size))
 
     dim_ordering = K.image_dim_ordering()

From 4d0368ad0559181ae65ebdec0f52c8b4c68ce954 Mon Sep 17 00:00:00 2001
From: Michael Oliver <michael.d.oliver@gmail.com>
Date: Wed, 22 Feb 2017 20:41:20 -0800
Subject: [PATCH 18/23] use consistent dim ordering

---
 tests/keras_contrib/layers/test_convolutional.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/tests/keras_contrib/layers/test_convolutional.py b/tests/keras_contrib/layers/test_convolutional.py
index 1120a7c..6469e2c 100644
--- a/tests/keras_contrib/layers/test_convolutional.py
+++ b/tests/keras_contrib/layers/test_convolutional.py
@@ -118,9 +118,6 @@ def test_cosineconvolution_2d():
                                    'dim_ordering': dim_ordering},
                            input_shape=(nb_samples, nb_row, nb_col, stack_size))
 
-    dim_ordering = K.image_dim_ordering()
-    assert dim_ordering in {'tf', 'th'}, 'dim_ordering must be in {tf, th}'
-
     if dim_ordering == 'th':
         X = np.random.randn(1, 3, 5, 5)
         input_dim = (3, 5, 5)
@@ -131,7 +128,7 @@ def test_cosineconvolution_2d():
         W0 = X[0, :, :, :, None]
 
     model = Sequential()
-    model.add(convolutional.CosineConvolution2D(1, 5, 5, bias=True, input_shape=input_dim))
+    model.add(convolutional.CosineConvolution2D(1, 5, 5, bias=True, input_shape=input_dim, dim_ordering=dim_ordering))
     model.compile(loss='mse', optimizer='rmsprop')
     W = model.get_weights()
     W[0] = W0
@@ -141,7 +138,7 @@ def test_cosineconvolution_2d():
     assert_allclose(out, np.ones((1, 1, 1, 1), dtype=K.floatx()), atol=1e-5)
 
     model = Sequential()
-    model.add(convolutional.CosineConvolution2D(1, 5, 5, bias=False, input_shape=input_dim))
+    model.add(convolutional.CosineConvolution2D(1, 5, 5, bias=False, input_shape=input_dim, dim_ordering=dim_ordering))
     model.compile(loss='mse', optimizer='rmsprop')
     W = model.get_weights()
     W[0] = -2 * W0

From 95ed5ad06db1578190c86a93723eec71565d2514 Mon Sep 17 00:00:00 2001
From: Michael Oliver <michael.d.oliver@gmail.com>
Date: Wed, 22 Feb 2017 21:14:37 -0800
Subject: [PATCH 19/23] clean up implementation

---
 keras_contrib/layers/core.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/keras_contrib/layers/core.py b/keras_contrib/layers/core.py
index bc142a5..b765c25 100644
--- a/keras_contrib/layers/core.py
+++ b/keras_contrib/layers/core.py
@@ -144,14 +144,16 @@ class CosineDense(Layer):
 
     def call(self, x, mask=None):
         if self.bias:
-            xnorm = K.sqrt(K.sum(K.square(x), axis=-1, keepdims=True) + 1 + K.epsilon())
-            x /= xnorm
-            Wnorm = K.sqrt(K.sum(K.square(self.W), axis=0) + K.square(self.b) + K.epsilon())
+            b, xb = self.b, 1.
         else:
-            x /= K.sqrt(K.sum(K.square(x), axis=-1, keepdims=True) + K.epsilon())
-            Wnorm = K.sqrt(K.sum(K.square(self.W), axis=0) + K.epsilon())
+            b, xb = 0., 0.
 
+        xnorm = K.sqrt(K.sum(K.square(x), axis=-1, keepdims=True) + xb + K.epsilon())
+        x /= xnorm
+
+        Wnorm = K.sqrt(K.sum(K.square(self.W), axis=0) + K.square(b) + K.epsilon())
         W = self.W / Wnorm
+
         output = K.dot(x, W)
         if self.bias:
             output += (self.b / (xnorm * Wnorm))

From 0eb0a0b08fdf23df43c29e3b9dab9bb7eedf8d89 Mon Sep 17 00:00:00 2001
From: tboquet <thomas.boquet@hec.ca>
Date: Sun, 26 Feb 2017 14:39:31 -0500
Subject: [PATCH 20/23] * doc and style fix

---
 keras_contrib/backend/tensorflow_backend.py | 40 +++++++++++----------
 1 file changed, 22 insertions(+), 18 deletions(-)

diff --git a/keras_contrib/backend/tensorflow_backend.py b/keras_contrib/backend/tensorflow_backend.py
index ec02c69..f33d7b2 100644
--- a/keras_contrib/backend/tensorflow_backend.py
+++ b/keras_contrib/backend/tensorflow_backend.py
@@ -58,7 +58,8 @@ def deconv3d(x, kernel, output_shape, strides=(1, 1, 1),
         raise ValueError('Unknown dim_ordering ' + str(dim_ordering))
 
     x = _preprocess_conv3d_input(x, dim_ordering)
-    output_shape = _preprocess_deconv_output_shape(x, output_shape, dim_ordering)
+    output_shape = _preprocess_deconv_output_shape(x, output_shape,
+                                                   dim_ordering)
     kernel = _preprocess_conv3d_kernel(kernel, dim_ordering)
     kernel = tf.transpose(kernel, (0, 1, 2, 4, 3))
     padding = _preprocess_border_mode(border_mode)
@@ -69,32 +70,35 @@ def deconv3d(x, kernel, output_shape, strides=(1, 1, 1),
     return _postprocess_conv3d_output(x, dim_ordering)
 
 
-def extract_image_patches(X, ksizes, ssizes, border_mode="same", dim_ordering="tf"):
+def extract_image_patches(x, ksizes, ssizes, border_mode="same",
+                          dim_ordering="tf"):
     '''
     Extract the patches from an image
-    Parameters
-    ----------
-    X : The input image
-    ksizes : 2-d tuple with the kernel size
-    ssizes : 2-d tuple with the strides size
-    border_mode : 'same' or 'valid'
-    dim_ordering : 'tf' or 'th'
-    Returns
-    -------
-    The (k_w,k_h) patches extracted
-    TF ==> (batch_size,w,h,k_w,k_h,c)
-    TH ==> (batch_size,w,h,c,k_w,k_h)
+    # Parameters
+
+        x : The input image
+        ksizes : 2-d tuple with the kernel size
+        ssizes : 2-d tuple with the strides size
+        border_mode : 'same' or 'valid'
+        dim_ordering : 'tf' or 'th'
+
+    # Returns
+        The (k_w,k_h) patches extracted
+        TF ==> (batch_size,w,h,k_w,k_h,c)
+        TH ==> (batch_size,w,h,c,k_w,k_h)
     '''
     kernel = [1, ksizes[0], ksizes[1], 1]
     strides = [1, ssizes[0], ssizes[1], 1]
     padding = _preprocess_border_mode(border_mode)
     if dim_ordering == "th":
-        X = KTF.permute_dimensions(X, (0, 2, 3, 1))
-    bs_i, w_i, h_i, ch_i = KTF.int_shape(X)
-    patches = tf.extract_image_patches(X, kernel, strides, [1, 1, 1, 1], padding)
+        x = KTF.permute_dimensions(x, (0, 2, 3, 1))
+    bs_i, w_i, h_i, ch_i = KTF.int_shape(x)
+    patches = tf.extract_image_patches(x, kernel, strides, [1, 1, 1, 1],
+                                       padding)
     # Reshaping to fit Theano
     bs, w, h, ch = KTF.int_shape(patches)
-    patches = tf.reshape(tf.transpose(tf.reshape(patches, [bs, w, h, -1, ch_i]), [0, 1, 2, 4, 3]),
+    patches = tf.reshape(tf.transpose(
+        tf.reshape(patches, [bs, w, h, -1, ch_i]), [0, 1, 2, 4, 3]),
                          [bs, w, h, ch_i, ksizes[0], ksizes[1]])
     if dim_ordering == "tf":
         patches = KTF.permute_dimensions(patches, [0, 1, 2, 4, 5, 3])

From b86deb141f446498f306f7b42e71eeb305d306aa Mon Sep 17 00:00:00 2001
From: tboquet <thomas.boquet@hec.ca>
Date: Sun, 26 Feb 2017 15:09:18 -0500
Subject: [PATCH 21/23] * break line in several operations

---
 keras_contrib/backend/tensorflow_backend.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/keras_contrib/backend/tensorflow_backend.py b/keras_contrib/backend/tensorflow_backend.py
index f33d7b2..4bac912 100644
--- a/keras_contrib/backend/tensorflow_backend.py
+++ b/keras_contrib/backend/tensorflow_backend.py
@@ -97,8 +97,8 @@ def extract_image_patches(x, ksizes, ssizes, border_mode="same",
                                        padding)
     # Reshaping to fit Theano
     bs, w, h, ch = KTF.int_shape(patches)
-    patches = tf.reshape(tf.transpose(
-        tf.reshape(patches, [bs, w, h, -1, ch_i]), [0, 1, 2, 4, 3]),
+    patches = tf.reshape(patches, [bs, w, h, -1, ch_i])
+    patches = tf.reshape(tf.transpose(patches, [0, 1, 2, 4, 3]),
                          [bs, w, h, ch_i, ksizes[0], ksizes[1]])
     if dim_ordering == "tf":
         patches = KTF.permute_dimensions(patches, [0, 1, 2, 4, 5, 3])

From 8db75c061967e01f2345b5b5c658be51e9b024d6 Mon Sep 17 00:00:00 2001
From: Michael Oliver <michael.d.oliver@gmail.com>
Date: Mon, 27 Feb 2017 14:19:52 -0800
Subject: [PATCH 22/23] move div to end for speed up

---
 keras_contrib/layers/core.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/keras_contrib/layers/core.py b/keras_contrib/layers/core.py
index b765c25..63c498e 100644
--- a/keras_contrib/layers/core.py
+++ b/keras_contrib/layers/core.py
@@ -149,14 +149,13 @@ class CosineDense(Layer):
             b, xb = 0., 0.
 
         xnorm = K.sqrt(K.sum(K.square(x), axis=-1, keepdims=True) + xb + K.epsilon())
-        x /= xnorm
-
         Wnorm = K.sqrt(K.sum(K.square(self.W), axis=0) + K.square(b) + K.epsilon())
-        W = self.W / Wnorm
 
-        output = K.dot(x, W)
+        xWnorm = (xnorm * Wnorm)
+
+        output = K.dot(x, self.W)/xWnorm
         if self.bias:
-            output += (self.b / (xnorm * Wnorm))
+            output += (self.b / xWnorm)
         return self.activation(output)
 
     def get_output_shape_for(self, input_shape):

From 603e5188bb1efc08058ef85b1775110219cd5031 Mon Sep 17 00:00:00 2001
From: Michael Oliver <michael.d.oliver@gmail.com>
Date: Mon, 27 Feb 2017 15:06:47 -0800
Subject: [PATCH 23/23] pep8 fix

---
 keras_contrib/layers/core.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/keras_contrib/layers/core.py b/keras_contrib/layers/core.py
index 63c498e..4c5da2c 100644
--- a/keras_contrib/layers/core.py
+++ b/keras_contrib/layers/core.py
@@ -153,7 +153,7 @@ class CosineDense(Layer):
 
         xWnorm = (xnorm * Wnorm)
 
-        output = K.dot(x, self.W)/xWnorm
+        output = K.dot(x, self.W) / xWnorm
         if self.bias:
             output += (self.b / xWnorm)
         return self.activation(output)