Merge commit '7dc329a5a3ec678d4c589529726ea5c7a6c458c8' into travis

* commit '7dc329a5a3ec678d4c589529726ea5c7a6c458c8':
  save_load_utils_test.py make tensorflow only
  Fix BatchRenormalization clipping (fixes #127)
  save_load_utils_test.py removes 'model.h5'
  recurrent.py pep8
  recurrent.py _time_distributed_dense doesn't exist
  setup.py update to match keras
  Fix for Python 3 in conll dataset loading

keras_contrib/backend/cntk_backend.py

@@ -1,2 +1,26 @@
 from keras.backend import cntk_backend as KCN
 import cntk as C
+import numpy as np
+
+
+def clip(x, min_value, max_value):
+    """Element-wise value clipping.
+
+    If min_value > max_value, clipping range is [min_value,min_value].
+
+    # Arguments
+        x: Tensor or variable.
+        min_value: Tensor, float, int, or None.
+            If min_value is None, defaults to -infinity.
+        max_value: Tensor, float, int, or None.
+            If max_value is None, defaults to infinity.
+
+    # Returns
+        A tensor.
+    """
+    if max_value is None:
+        max_value = np.inf
+    if min_value is None:
+        min_value = -np.inf
+    max_value = C.maximum(min_value, max_value)
+    return C.clip(x, min_value, max_value)

keras_contrib/backend/tensorflow_backend.py

@@ -1,4 +1,5 @@
 import tensorflow as tf
+import numpy as np
 
 try:
     from tensorflow.python.ops import ctc_ops as ctc
@@ -11,6 +12,7 @@ from keras.backend.tensorflow_backend import _postprocess_conv3d_output
 from keras.backend.tensorflow_backend import _preprocess_padding
 from keras.backend.tensorflow_backend import _preprocess_conv2d_input
 from keras.backend.tensorflow_backend import _postprocess_conv2d_output
+from keras.backend.tensorflow_backend import _to_tensor
 
 py_all = all
 
@@ -158,3 +160,28 @@ def moments(x, axes, shift=None, keep_dims=False):
     ''' Wrapper over tensorflow backend call '''
 
     return tf.nn.moments(x, axes, shift=shift, keep_dims=keep_dims)
+
+
+def clip(x, min_value, max_value):
+    """Element-wise value clipping.
+
+    If min_value > max_value, clipping range is [min_value,min_value].
+
+    # Arguments
+        x: Tensor or variable.
+        min_value: Tensor, float, int, or None.
+            If min_value is None, defaults to -infinity.
+        max_value: Tensor, float, int, or None.
+            If max_value is None, defaults to infinity.
+
+    # Returns
+        A tensor.
+    """
+    if max_value is None:
+        max_value = np.inf
+    if min_value is None:
+        min_value = -np.inf
+    min_value = _to_tensor(min_value, x.dtype.base_dtype)
+    max_value = _to_tensor(max_value, x.dtype.base_dtype)
+    max_value = tf.maximum(min_value, max_value)
+    return tf.clip_by_value(x, min_value, max_value)

keras_contrib/backend/theano_backend.py

@@ -1,5 +1,6 @@
 from theano import tensor as T
 from theano.sandbox.neighbours import images2neibs
+import numpy as np
 
 try:
     import theano.sparse as th_sparse_module
@@ -197,3 +198,26 @@ def moments(x, axes, shift=None, keep_dims=False):
     var_batch = KTH.var(x, axis=axes, keepdims=keep_dims)
 
     return mean_batch, var_batch
+
+
+def clip(x, min_value, max_value):
+    """Element-wise value clipping.
+
+    If min_value > max_value, clipping range is [min_value,min_value].
+
+    # Arguments
+        x: Tensor or variable.
+        min_value: Tensor, float, int, or None.
+            If min_value is None, defaults to -infinity.
+        max_value: Tensor, float, int, or None.
+            If max_value is None, defaults to infinity.
+
+    # Returns
+        A tensor.
+    """
+    if max_value is None:
+        max_value = np.inf
+    if min_value is None:
+        min_value = -np.inf
+    max_value = T.maximum(min_value, max_value)
+    return T.clip(x, min_value, max_value)

keras_contrib/datasets/conll2000.py

@@ -16,7 +16,7 @@ def load_data(path='conll2000.zip', min_freq=2):
     archive.close()
 
     word_counts = Counter(row[0].lower() for sample in train for row in sample)
-    vocab = ['<pad>', '<unk>'] + [w for w, f in word_counts.iteritems() if f >= min_freq]
+    vocab = ['<pad>', '<unk>'] + [w for w, f in iter(word_counts.items()) if f >= min_freq]
     pos_tags = sorted(list(set(row[1] for sample in train + test for row in sample)))  # in alphabetic order
     chunk_tags = sorted(list(set(row[2] for sample in train + test for row in sample)))  # in alphabetic order
 
@@ -27,7 +27,7 @@ def load_data(path='conll2000.zip', min_freq=2):
 
 def _parse_data(fh):
     string = fh.read()
-    data = [[row.split() for row in sample.split('\n')] for sample in string.strip().split('\n\n')]
+    data = [[row.split() for row in sample.split('\n')] for sample in string.decode().strip().split('\n\n')]
     fh.close()
     return data
 

keras_contrib/layers/normalization.py

@@ -266,13 +266,13 @@ class BatchRenormalization(Layer):
                                                 name='{}_running_std'.format(self.name),
                                                 trainable=False)
 
-        self.r_max = K.variable(np.ones((1,)), name='{}_r_max'.format(self.name))
+        self.r_max = K.variable(1, name='{}_r_max'.format(self.name))
 
-        self.d_max = K.variable(np.zeros((1,)), name='{}_d_max'.format(self.name))
+        self.d_max = K.variable(0, name='{}_d_max'.format(self.name))
 
-        self.t = K.variable(np.zeros((1,)), name='{}_t'.format(self.name))
+        self.t = K.variable(0, name='{}_t'.format(self.name))
 
-        self.t_delta_tensor = K.variable(np.array([self.t_delta]))
+        self.t_delta_tensor = K.constant(self.t_delta)
 
         if self.initial_weights is not None:
             self.set_weights(self.initial_weights)
@@ -292,13 +292,11 @@ class BatchRenormalization(Layer):
         mean_batch, var_batch = K.moments(inputs, reduction_axes, shift=None, keep_dims=False)
         std_batch = (K.sqrt(var_batch + self.epsilon))
 
-        r_max_value = K.get_value(self.r_max)
         r = std_batch / (K.sqrt(self.running_variance + self.epsilon))
-        r = K.stop_gradient(K.clip(r, 1 / r_max_value, r_max_value))
+        r = K.stop_gradient(K.clip(r, 1 / self.r_max, self.r_max))
 
-        d_max_value = K.get_value(self.d_max)
         d = (mean_batch - self.running_mean) / K.sqrt(self.running_variance + self.epsilon)
-        d = K.stop_gradient(K.clip(d, -d_max_value, d_max_value))
+        d = K.stop_gradient(K.clip(d, -self.d_max, self.d_max))
 
         if sorted(reduction_axes) == range(K.ndim(inputs))[:-1]:
             x_normed_batch = (inputs - mean_batch) / std_batch

keras_contrib/layers/recurrent.py

@@ -8,5 +8,3 @@ from .. import initializers
 from .. import regularizers
 from keras.engine import Layer
 from keras.engine import InputSpec
-
-from keras.layers.recurrent import _time_distributed_dense

setup.py

@@ -3,11 +3,32 @@ from setuptools import find_packages
 
 
 setup(name='keras_contrib',
-      version='1.2.1',
-      description='Keras community contributions',
+      version='2.0.8',
+      description='Keras Deep Learning for Python, Community Contributions',
       author='Fariz Rahman',
       author_email='farizrahman4u@gmail.com',
       url='https://github.com/farizrahman4u/keras-contrib',
       license='MIT',
       install_requires=['keras'],
+      extras_require={
+          'h5py': ['h5py'],
+          'visualize': ['pydot>=1.2.0'],
+          'tests': ['pytest',
+                    'pytest-pep8',
+                    'pytest-xdist',
+                    'pytest-cov'],
+      },
+      classifiers=[
+          'Development Status :: 3 - Alpha',
+          'Intended Audience :: Developers',
+          'Intended Audience :: Education',
+          'Intended Audience :: Science/Research',
+          'License :: OSI Approved :: MIT License',
+          'Programming Language :: Python :: 2',
+          'Programming Language :: Python :: 2.7',
+          'Programming Language :: Python :: 3',
+          'Programming Language :: Python :: 3.6',
+          'Topic :: Software Development :: Libraries',
+          'Topic :: Software Development :: Libraries :: Python Modules'
+      ],
       packages=find_packages())

tests/keras_contrib/backend/backend_test.py

@@ -160,6 +160,43 @@ class TestBackend(object):
                     assert_allclose(th_mean_val, tf_mean_val, rtol=1e-4)
                     assert_allclose(th_var_val, tf_var_val, rtol=1e-4)
 
+    def test_clip(self):
+        check_single_tensor_operation('clip', (4, 2), min_value=0.4, max_value=0.6)
+        check_single_tensor_operation('clip', (4, 2), min_value=0.4, max_value=None)
+
+        cases = [
+            # (x, min_value, max_value, expected)
+            (1, 0, 2, 1),
+            (1, 2, 0, 2),
+            (-1, 0, 2, 0),
+            (-1, 2, 0, 2),
+            (3, 0, 2, 2),
+            (3, 2, 0, 2),
+            (1, 0, np.inf, 1),
+            (1, np.inf, 0, np.inf),
+            (1, 0, -np.inf, 0),
+            (1, -np.inf, 0, 0),
+            (-1, 0, -np.inf, 0),
+            (-1, -np.inf, 0, -1),
+            (1, 0, None, 1),
+            (-1, 0, None, 0),
+
+            # NOTE: In the following two cases, Keras 2.0.8 raises an
+            # error on all backends, but this is a sensible extension.
+            (1, None, 0, 0),
+            (-1, None, 0, -1),
+
+            # NOTE: In the following case, Keras 2.0.8 rasies an error
+            # for TensorFlow and Theano, but returns 0 for CNTK. This
+            # extends the TensorFlow and Theano backends to match the
+            # CNTK behavior instead of raising an error.
+            (0, None, None, 0),
+        ]
+        for K_, KC_ in [(KTF, KCTF), (KTH, KCTH)]:
+            for x, min_value, max_value, expected in cases:
+                actual = K_.eval(KC_.clip(K_.constant(x), min_value, max_value))
+                assert_allclose(expected, actual, atol=1e-5)
+
 
 if __name__ == '__main__':
     pytest.main([__file__])

tests/keras_contrib/layers/test_normalization.py

@@ -305,5 +305,37 @@ def test_shared_batchrenorm():
     new_model.train_on_batch(x, x)
 
 
+@keras_test
+def test_batchrenorm_clipping_schedule():
+    '''Test that the clipping schedule isn't fixed at r_max=1, d_max=0'''
+    inp = Input(shape=(10,))
+    bn = normalization.BatchRenormalization(t_delta=1.)
+    out = bn(inp)
+    model = Model(inp, out)
+    model.compile('sgd', 'mse')
+
+    x = np.random.normal(5, 10, size=(2, 10))
+    y = np.random.normal(5, 10, size=(2, 10))
+
+    r_max, d_max = K.get_value(bn.r_max), K.get_value(bn.d_max)
+    assert r_max == 1
+    assert d_max == 0
+
+    for i in range(10):
+        model.train_on_batch(x, y)
+
+    r_max, d_max = K.get_value(bn.r_max), K.get_value(bn.d_max)
+    assert_allclose([r_max, d_max], [3, 5], atol=1e-1)
+
+
+@keras_test
+def test_batchrenorm_get_config():
+    '''Test that get_config works on a model with a batchrenorm layer.'''
+    x = Input(shape=(10,))
+    y = normalization.BatchRenormalization()(x)
+    model = Model(x, y)
+    model.get_config()
+
+
 if __name__ == '__main__':
     pytest.main([__file__])

tests/keras_contrib/utils/save_load_utils_test.py

@@ -1,12 +1,16 @@
 import pytest
+import os
 from keras import backend as K
 from keras.layers import Input, Dense
 from keras.models import Model
 from numpy.testing import assert_allclose
+from keras.utils.test_utils import keras_test
 
 from keras_contrib.utils.save_load_utils import save_all_weights, load_all_weights
 
 
+@pytest.mark.skipif(K.backend() != 'tensorflow', reason='save_all_weights and load_all_weights only supported on TensorFlow')
+@keras_test
 def test_save_and_load_all_weights():
     '''
     Test save_all_weights and load_all_weights. Save and load optimizer and model weights but not configuration.
@@ -33,15 +37,16 @@ def test_save_and_load_all_weights():
     ow1value[0, 0:3] = [4, 2, 0]
     K.set_value(ow1, ow1value)
     # save all weights
-    save_all_weights(m1, "model.h5")
+    save_all_weights(m1, 'model.h5')
     # new model
     m2 = make_model()
     # load all weights
-    load_all_weights(m2, "model.h5")
+    load_all_weights(m2, 'model.h5')
     # check weights
     assert_allclose(K.get_value(m2.layers[1].kernel)[0, 0:4], [1, 3, 3, 7])
     # check optimizer weights
     assert_allclose(K.get_value(m2.optimizer.weights[3])[0, 0:3], [4, 2, 0])
+    os.remove('model.h5')
 
 
 if __name__ == '__main__':