diff --git a/keras_contrib/backend/cntk_backend.py b/keras_contrib/backend/cntk_backend.py index 363ad08..624aeee 100644 --- a/keras_contrib/backend/cntk_backend.py +++ b/keras_contrib/backend/cntk_backend.py @@ -1,2 +1,26 @@ from keras.backend import cntk_backend as KCN import cntk as C +import numpy as np + + +def clip(x, min_value, max_value): + """Element-wise value clipping. + + If min_value > max_value, clipping range is [min_value,min_value]. + + # Arguments + x: Tensor or variable. + min_value: Tensor, float, int, or None. + If min_value is None, defaults to -infinity. + max_value: Tensor, float, int, or None. + If max_value is None, defaults to infinity. + + # Returns + A tensor. + """ + if max_value is None: + max_value = np.inf + if min_value is None: + min_value = -np.inf + max_value = C.maximum(min_value, max_value) + return C.clip(x, min_value, max_value) diff --git a/keras_contrib/backend/tensorflow_backend.py b/keras_contrib/backend/tensorflow_backend.py index 7b69687..82b3c7f 100644 --- a/keras_contrib/backend/tensorflow_backend.py +++ b/keras_contrib/backend/tensorflow_backend.py @@ -1,4 +1,5 @@ import tensorflow as tf +import numpy as np try: from tensorflow.python.ops import ctc_ops as ctc @@ -11,6 +12,7 @@ from keras.backend.tensorflow_backend import _postprocess_conv3d_output from keras.backend.tensorflow_backend import _preprocess_padding from keras.backend.tensorflow_backend import _preprocess_conv2d_input from keras.backend.tensorflow_backend import _postprocess_conv2d_output +from keras.backend.tensorflow_backend import _to_tensor py_all = all @@ -158,3 +160,28 @@ def moments(x, axes, shift=None, keep_dims=False): ''' Wrapper over tensorflow backend call ''' return tf.nn.moments(x, axes, shift=shift, keep_dims=keep_dims) + + +def clip(x, min_value, max_value): + """Element-wise value clipping. + + If min_value > max_value, clipping range is [min_value,min_value]. + + # Arguments + x: Tensor or variable. + min_value: Tensor, float, int, or None. + If min_value is None, defaults to -infinity. + max_value: Tensor, float, int, or None. + If max_value is None, defaults to infinity. + + # Returns + A tensor. + """ + if max_value is None: + max_value = np.inf + if min_value is None: + min_value = -np.inf + min_value = _to_tensor(min_value, x.dtype.base_dtype) + max_value = _to_tensor(max_value, x.dtype.base_dtype) + max_value = tf.maximum(min_value, max_value) + return tf.clip_by_value(x, min_value, max_value) diff --git a/keras_contrib/backend/theano_backend.py b/keras_contrib/backend/theano_backend.py index 2b5adaf..9e97084 100644 --- a/keras_contrib/backend/theano_backend.py +++ b/keras_contrib/backend/theano_backend.py @@ -1,5 +1,6 @@ from theano import tensor as T from theano.sandbox.neighbours import images2neibs +import numpy as np try: import theano.sparse as th_sparse_module @@ -197,3 +198,26 @@ def moments(x, axes, shift=None, keep_dims=False): var_batch = KTH.var(x, axis=axes, keepdims=keep_dims) return mean_batch, var_batch + + +def clip(x, min_value, max_value): + """Element-wise value clipping. + + If min_value > max_value, clipping range is [min_value,min_value]. + + # Arguments + x: Tensor or variable. + min_value: Tensor, float, int, or None. + If min_value is None, defaults to -infinity. + max_value: Tensor, float, int, or None. + If max_value is None, defaults to infinity. + + # Returns + A tensor. + """ + if max_value is None: + max_value = np.inf + if min_value is None: + min_value = -np.inf + max_value = T.maximum(min_value, max_value) + return T.clip(x, min_value, max_value) diff --git a/keras_contrib/datasets/conll2000.py b/keras_contrib/datasets/conll2000.py old mode 100644 new mode 100755 index 22a97e1..5561f17 --- a/keras_contrib/datasets/conll2000.py +++ b/keras_contrib/datasets/conll2000.py @@ -16,7 +16,7 @@ def load_data(path='conll2000.zip', min_freq=2): archive.close() word_counts = Counter(row[0].lower() for sample in train for row in sample) - vocab = ['', ''] + [w for w, f in word_counts.iteritems() if f >= min_freq] + vocab = ['', ''] + [w for w, f in iter(word_counts.items()) if f >= min_freq] pos_tags = sorted(list(set(row[1] for sample in train + test for row in sample))) # in alphabetic order chunk_tags = sorted(list(set(row[2] for sample in train + test for row in sample))) # in alphabetic order @@ -27,7 +27,7 @@ def load_data(path='conll2000.zip', min_freq=2): def _parse_data(fh): string = fh.read() - data = [[row.split() for row in sample.split('\n')] for sample in string.strip().split('\n\n')] + data = [[row.split() for row in sample.split('\n')] for sample in string.decode().strip().split('\n\n')] fh.close() return data diff --git a/keras_contrib/layers/normalization.py b/keras_contrib/layers/normalization.py index 10565b2..40254cd 100644 --- a/keras_contrib/layers/normalization.py +++ b/keras_contrib/layers/normalization.py @@ -266,13 +266,13 @@ class BatchRenormalization(Layer): name='{}_running_std'.format(self.name), trainable=False) - self.r_max = K.variable(np.ones((1,)), name='{}_r_max'.format(self.name)) + self.r_max = K.variable(1, name='{}_r_max'.format(self.name)) - self.d_max = K.variable(np.zeros((1,)), name='{}_d_max'.format(self.name)) + self.d_max = K.variable(0, name='{}_d_max'.format(self.name)) - self.t = K.variable(np.zeros((1,)), name='{}_t'.format(self.name)) + self.t = K.variable(0, name='{}_t'.format(self.name)) - self.t_delta_tensor = K.variable(np.array([self.t_delta])) + self.t_delta_tensor = K.constant(self.t_delta) if self.initial_weights is not None: self.set_weights(self.initial_weights) @@ -292,13 +292,11 @@ class BatchRenormalization(Layer): mean_batch, var_batch = K.moments(inputs, reduction_axes, shift=None, keep_dims=False) std_batch = (K.sqrt(var_batch + self.epsilon)) - r_max_value = K.get_value(self.r_max) r = std_batch / (K.sqrt(self.running_variance + self.epsilon)) - r = K.stop_gradient(K.clip(r, 1 / r_max_value, r_max_value)) + r = K.stop_gradient(K.clip(r, 1 / self.r_max, self.r_max)) - d_max_value = K.get_value(self.d_max) d = (mean_batch - self.running_mean) / K.sqrt(self.running_variance + self.epsilon) - d = K.stop_gradient(K.clip(d, -d_max_value, d_max_value)) + d = K.stop_gradient(K.clip(d, -self.d_max, self.d_max)) if sorted(reduction_axes) == range(K.ndim(inputs))[:-1]: x_normed_batch = (inputs - mean_batch) / std_batch diff --git a/keras_contrib/layers/recurrent.py b/keras_contrib/layers/recurrent.py index e85dc22..c85a6c6 100644 --- a/keras_contrib/layers/recurrent.py +++ b/keras_contrib/layers/recurrent.py @@ -8,5 +8,3 @@ from .. import initializers from .. import regularizers from keras.engine import Layer from keras.engine import InputSpec - -from keras.layers.recurrent import _time_distributed_dense diff --git a/setup.py b/setup.py index aad0567..3c17537 100644 --- a/setup.py +++ b/setup.py @@ -3,11 +3,32 @@ from setuptools import find_packages setup(name='keras_contrib', - version='1.2.1', - description='Keras community contributions', + version='2.0.8', + description='Keras Deep Learning for Python, Community Contributions', author='Fariz Rahman', author_email='farizrahman4u@gmail.com', url='https://github.com/farizrahman4u/keras-contrib', license='MIT', install_requires=['keras'], + extras_require={ + 'h5py': ['h5py'], + 'visualize': ['pydot>=1.2.0'], + 'tests': ['pytest', + 'pytest-pep8', + 'pytest-xdist', + 'pytest-cov'], + }, + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'Intended Audience :: Education', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules' + ], packages=find_packages()) diff --git a/tests/keras_contrib/backend/backend_test.py b/tests/keras_contrib/backend/backend_test.py index 64135b0..998bc6b 100644 --- a/tests/keras_contrib/backend/backend_test.py +++ b/tests/keras_contrib/backend/backend_test.py @@ -160,6 +160,43 @@ class TestBackend(object): assert_allclose(th_mean_val, tf_mean_val, rtol=1e-4) assert_allclose(th_var_val, tf_var_val, rtol=1e-4) + def test_clip(self): + check_single_tensor_operation('clip', (4, 2), min_value=0.4, max_value=0.6) + check_single_tensor_operation('clip', (4, 2), min_value=0.4, max_value=None) + + cases = [ + # (x, min_value, max_value, expected) + (1, 0, 2, 1), + (1, 2, 0, 2), + (-1, 0, 2, 0), + (-1, 2, 0, 2), + (3, 0, 2, 2), + (3, 2, 0, 2), + (1, 0, np.inf, 1), + (1, np.inf, 0, np.inf), + (1, 0, -np.inf, 0), + (1, -np.inf, 0, 0), + (-1, 0, -np.inf, 0), + (-1, -np.inf, 0, -1), + (1, 0, None, 1), + (-1, 0, None, 0), + + # NOTE: In the following two cases, Keras 2.0.8 raises an + # error on all backends, but this is a sensible extension. + (1, None, 0, 0), + (-1, None, 0, -1), + + # NOTE: In the following case, Keras 2.0.8 rasies an error + # for TensorFlow and Theano, but returns 0 for CNTK. This + # extends the TensorFlow and Theano backends to match the + # CNTK behavior instead of raising an error. + (0, None, None, 0), + ] + for K_, KC_ in [(KTF, KCTF), (KTH, KCTH)]: + for x, min_value, max_value, expected in cases: + actual = K_.eval(KC_.clip(K_.constant(x), min_value, max_value)) + assert_allclose(expected, actual, atol=1e-5) + if __name__ == '__main__': pytest.main([__file__]) diff --git a/tests/keras_contrib/layers/test_normalization.py b/tests/keras_contrib/layers/test_normalization.py index fe2a172..b63f2c7 100644 --- a/tests/keras_contrib/layers/test_normalization.py +++ b/tests/keras_contrib/layers/test_normalization.py @@ -305,5 +305,37 @@ def test_shared_batchrenorm(): new_model.train_on_batch(x, x) +@keras_test +def test_batchrenorm_clipping_schedule(): + '''Test that the clipping schedule isn't fixed at r_max=1, d_max=0''' + inp = Input(shape=(10,)) + bn = normalization.BatchRenormalization(t_delta=1.) + out = bn(inp) + model = Model(inp, out) + model.compile('sgd', 'mse') + + x = np.random.normal(5, 10, size=(2, 10)) + y = np.random.normal(5, 10, size=(2, 10)) + + r_max, d_max = K.get_value(bn.r_max), K.get_value(bn.d_max) + assert r_max == 1 + assert d_max == 0 + + for i in range(10): + model.train_on_batch(x, y) + + r_max, d_max = K.get_value(bn.r_max), K.get_value(bn.d_max) + assert_allclose([r_max, d_max], [3, 5], atol=1e-1) + + +@keras_test +def test_batchrenorm_get_config(): + '''Test that get_config works on a model with a batchrenorm layer.''' + x = Input(shape=(10,)) + y = normalization.BatchRenormalization()(x) + model = Model(x, y) + model.get_config() + + if __name__ == '__main__': pytest.main([__file__]) diff --git a/tests/keras_contrib/utils/save_load_utils_test.py b/tests/keras_contrib/utils/save_load_utils_test.py index 67f55fc..a11e826 100644 --- a/tests/keras_contrib/utils/save_load_utils_test.py +++ b/tests/keras_contrib/utils/save_load_utils_test.py @@ -1,12 +1,16 @@ import pytest +import os from keras import backend as K from keras.layers import Input, Dense from keras.models import Model from numpy.testing import assert_allclose +from keras.utils.test_utils import keras_test from keras_contrib.utils.save_load_utils import save_all_weights, load_all_weights +@pytest.mark.skipif(K.backend() != 'tensorflow', reason='save_all_weights and load_all_weights only supported on TensorFlow') +@keras_test def test_save_and_load_all_weights(): ''' Test save_all_weights and load_all_weights. Save and load optimizer and model weights but not configuration. @@ -33,15 +37,16 @@ def test_save_and_load_all_weights(): ow1value[0, 0:3] = [4, 2, 0] K.set_value(ow1, ow1value) # save all weights - save_all_weights(m1, "model.h5") + save_all_weights(m1, 'model.h5') # new model m2 = make_model() # load all weights - load_all_weights(m2, "model.h5") + load_all_weights(m2, 'model.h5') # check weights assert_allclose(K.get_value(m2.layers[1].kernel)[0, 0:4], [1, 3, 3, 7]) # check optimizer weights assert_allclose(K.get_value(m2.optimizer.weights[3])[0, 0:3], [4, 2, 0]) + os.remove('model.h5') if __name__ == '__main__':