From 64ee9acfdbef6fe519ed17decd2aebdb778e64b0 Mon Sep 17 00:00:00 2001 From: lameeus Date: Wed, 25 Oct 2017 12:13:09 +0200 Subject: [PATCH 01/16] 1) Extended layers to everything that has a ReLU activation 2) Added layer name to the warning 3) Corrected the dead neuron count (should work for both Theano and TF (tested for TF)) 4) Added a print alongside the warning since for me the warning stopped showing after some epochs (might be a bug at my side). --- keras_contrib/callbacks/dead_relu_detector.py | 54 ++++++++++++++----- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/keras_contrib/callbacks/dead_relu_detector.py b/keras_contrib/callbacks/dead_relu_detector.py index 2019f56..6ad2074 100644 --- a/keras_contrib/callbacks/dead_relu_detector.py +++ b/keras_contrib/callbacks/dead_relu_detector.py @@ -2,7 +2,6 @@ import numpy as np import warnings from keras.callbacks import Callback -from keras.layers import Dense from keras import backend as K @@ -17,6 +16,7 @@ class DeadReluDetector(Callback): False means that only significant number of dead neurons (10% or more) triggers warning """ + def __init__(self, x_train, verbose=False): super(DeadReluDetector, self).__init__() self.x_train = x_train @@ -25,7 +25,11 @@ class DeadReluDetector(Callback): @staticmethod def is_relu_layer(layer): - return isinstance(layer, Dense) and layer.get_config()['activation'] == 'relu' + # Should work for all layers with relu activation. Tested for Dense and Conv2D + if 'activation' in layer.get_config(): + return layer.get_config()['activation'] == 'relu' + else: + return False def get_relu_activations(self): model_input = self.model.input @@ -44,17 +48,43 @@ class DeadReluDetector(Callback): layer_outputs = [func(list_inputs)[0] for func in funcs] for layer_index, layer_activations in enumerate(layer_outputs): if self.is_relu_layer(self.model.layers[layer_index]): - yield [layer_index, layer_activations] + layer_name = self.model.layers[layer_index].name + # layer_weight is a list [W] (+ [b]) + layer_weight = self.model.layers[layer_index].get_weights() + # with kernel and bias, the weights are saved as a list [W, b]. If only weights, it is [W] + assert type(layer_weight) == list + layer_weight_shape = np.shape(layer_weight[0]) + yield [layer_index, layer_activations, layer_name, layer_weight_shape] def on_epoch_end(self, epoch, logs={}): for relu_activation in self.get_relu_activations(): - layer_index, activation_values = relu_activation - total_neurons = activation_values.shape[-1] - dead_neurons = np.sum(activation_values == 0) - dead_neurons_share = dead_neurons / total_neurons + layer_index, activation_values, layer_name, layer_weight_shape = relu_activation + + shape_act = activation_values.shape + + weight_len = len(layer_weight_shape) + act_len = len(shape_act) + + # should work for both Conv and Flat + if K.backend() == 'tensorflow': + # features in last axis + axis_filter = -1 + elif K.backend() == 'theano': + # features before the convolution axis, for weight_len the input and output have to be subtracted + axis_filter = -1 - (weight_len - 2) + else: + raise ValueError('Unknown backend: {}'.format(K.backend())) + + total_featuremaps = shape_act[axis_filter] + + axis = tuple( + i for i in range(act_len) if (i != axis_filter) and (i != (len(shape_act) + axis_filter))) + + dead_neurons = np.sum(np.sum(activation_values, axis=axis) == 0) + + dead_neurons_share = dead_neurons / total_featuremaps if (self.verbose and dead_neurons > 0) or dead_neurons_share > self.dead_neurons_share_threshold: - warnings.warn( - 'Layer #{} has {} dead neurons ({:.2%})!' - .format(layer_index, dead_neurons, dead_neurons_share), - RuntimeWarning - ) + str_warning = 'Layer {} (#{}) has {} dead neurons ({:.2%})!'.format(layer_name, layer_index, + dead_neurons, dead_neurons_share) + print(str_warning) + warnings.warn(str_warning, RuntimeWarning) From d635747da3cfe5deb5782401139d1942f029faeb Mon Sep 17 00:00:00 2001 From: lameeus Date: Wed, 25 Oct 2017 12:14:39 +0200 Subject: [PATCH 02/16] Made first test more general + test if bias included + test when using a convolutional layer (Conv2D) --- .../callbacks/dead_relu_detector_test.py | 123 ++++++++++++++++-- 1 file changed, 115 insertions(+), 8 deletions(-) diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index 9a37df9..89c6bbe 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -4,19 +4,30 @@ import numpy as np from keras_contrib import callbacks from keras.models import Sequential -from keras.layers import Dense +from keras.layers import Dense, Conv2D, Flatten +from keras import backend as K def test_DeadDeadReluDetector(): + n_samples = 9 + + input_shape = (n_samples, 3, 4) # 4 input features + shape_out = (n_samples, 3, 10) # 10 output features + shape_weights = (4, 10) + + # ignore batch size + input_shape_dense = tuple(input_shape[1:]) + def do_test(weights, expected_warnings, verbose): with warnings.catch_warnings(record=True) as w: - dataset = np.ones((1, 1, 1)) # data to be fed as training + dataset = np.ones(input_shape) # data to be fed as training model = Sequential() - model.add(Dense(10, activation='relu', input_shape=(1, 1), use_bias=False, weights=[weights])) + model.add(Dense(10, activation='relu', input_shape=input_shape_dense, + use_bias=False, weights=[weights], name='dense')) model.compile(optimizer='sgd', loss='categorical_crossentropy') model.fit( dataset, - np.ones((1, 1, 10)), + np.ones(shape_out), epochs=1, callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], verbose=False @@ -26,16 +37,112 @@ def test_DeadDeadReluDetector(): assert issubclass(warn_item.category, RuntimeWarning) assert "dead neurons" in str(warn_item.message) - weights_1_dead = np.ones((1, 10)) # weights that correspond to NN with 1/10 neurons dead + weights_1_dead = np.ones(shape_weights) # weights that correspond to NN with 1/10 neurons dead + weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/10 neurons dead + weights_1_dead[:, 0] = 0 - weights_2_dead = np.ones((1, 10)) # weights that correspond to NN with 2/10 neurons dead - weights_2_dead[:, 0] = 0 - weights_2_dead[:, 1] = 0 + weights_2_dead[:, 0:2] = 0 do_test(weights_1_dead, verbose=True, expected_warnings=1) do_test(weights_1_dead, verbose=False, expected_warnings=0) do_test(weights_2_dead, verbose=True, expected_warnings=1) +def test_DeadDeadReluDetector_bias(): + n_samples = 9 + + input_shape = (n_samples, 4) # 4 input features + shape_weights = (4, 10) + shape_out = (n_samples, 10) # 10 output features + shape_bias = (10, ) + + # ignore batch size + input_shape_dense = tuple(input_shape[1:]) + + def do_test(weights, bias, expected_warnings, verbose): + with warnings.catch_warnings(record=True) as w: + dataset = np.ones(input_shape) # data to be fed as training + model = Sequential() + model.add(Dense(10, activation='relu', input_shape=input_shape_dense, + use_bias=True, weights=[weights, bias], name='dense')) + model.compile(optimizer='sgd', loss='categorical_crossentropy') + # model.compile(optimizer=None, loss='categorical_crossentropy') + model.fit( + dataset, + np.ones(shape_out), + epochs=1, + callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], + verbose=False + ) + assert len(w) == expected_warnings + for warn_item in w: + assert issubclass(warn_item.category, RuntimeWarning) + assert "dead neurons" in str(warn_item.message) + + weights_1_dead = np.ones(shape_weights) # weights that correspond to NN with 1/10 neurons dead + weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/10 neurons dead + + weights_1_dead[:, 0] = 0 + weights_2_dead[:, 0:2] = 0 + + bias = np.zeros(shape_bias) + + do_test(weights_1_dead, bias, verbose=True, expected_warnings=1) + do_test(weights_1_dead, bias, verbose=False, expected_warnings=0) + do_test(weights_2_dead, bias, verbose=True, expected_warnings=1) + + +def test_DeadDeadReluDetector_conv(): + n_samples = 9 + + # (5, 5) kernel, 4 input featuremaps and 10 output featuremaps + if K.backend() == 'tensorflow': + input_shape = (n_samples, 5, 5, 4) + elif K.backend() == 'theano': + input_shape = (n_samples, 4, 5, 5) + else: + raise ValueError('Unknown backend: {}'.format(K.backend())) + + # ignore batch size + input_shape_conv = tuple(input_shape[1:]) + shape_weights = (5, 5, 4, 10) + shape_out = (n_samples, 10) + + def do_test(weights_bias, expected_warnings, verbose): + with warnings.catch_warnings(record=True) as w: + + dataset = np.ones(input_shape) # data to be fed as training + model = Sequential() + model.add(Conv2D(10, (5, 5), activation='relu', input_shape=input_shape_conv, + use_bias=True, weights=weights_bias, name='conv')) + model.add(Flatten()) # to handle Theano's categorical crossentropy + model.compile(optimizer='sgd', loss='categorical_crossentropy') + model.fit( + dataset, + np.ones(shape_out), + epochs=1, + callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], + verbose=False + ) + assert len(w) == expected_warnings + for warn_item in w: + assert issubclass(warn_item.category, RuntimeWarning) + assert "dead neurons" in str(warn_item.message) + + weights_1_dead = np.ones(shape_weights) # weights that correspond to NN with 1/10 neurons dead + weights_1_dead[..., 0] = 0 + weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/10 neurons dead + weights_2_dead[..., 0:2] = 0 + + bias = np.zeros((10, )) + + weights_bias_1_dead = [weights_1_dead, bias] + weights_bias_2_dead = [weights_2_dead, bias] + + do_test(weights_bias_1_dead, verbose=True, expected_warnings=1) + do_test(weights_bias_1_dead, verbose=False, expected_warnings=0) + do_test(weights_bias_2_dead, verbose=True, expected_warnings=1) + + if __name__ == '__main__': pytest.main([__file__]) From 3d710d4984cfd6a4462642221ad4c7ac838744ba Mon Sep 17 00:00:00 2001 From: lameeus Date: Wed, 25 Oct 2017 13:32:01 +0200 Subject: [PATCH 03/16] Simplification (suggested by @arodiss) --- keras_contrib/callbacks/dead_relu_detector.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/keras_contrib/callbacks/dead_relu_detector.py b/keras_contrib/callbacks/dead_relu_detector.py index 6ad2074..868fc16 100644 --- a/keras_contrib/callbacks/dead_relu_detector.py +++ b/keras_contrib/callbacks/dead_relu_detector.py @@ -26,10 +26,7 @@ class DeadReluDetector(Callback): @staticmethod def is_relu_layer(layer): # Should work for all layers with relu activation. Tested for Dense and Conv2D - if 'activation' in layer.get_config(): - return layer.get_config()['activation'] == 'relu' - else: - return False + return 'activation' in layer.get_config() and layer.get_config()['activation'] == 'relu' def get_relu_activations(self): model_input = self.model.input From ececa1b7afe09e3566ba2f19004aa24a655dd52f Mon Sep 17 00:00:00 2001 From: lameeus Date: Wed, 25 Oct 2017 13:45:52 +0200 Subject: [PATCH 04/16] Made indepent of backend. Now only checks channel priority (as suggested by @arodiss) --- keras_contrib/callbacks/dead_relu_detector.py | 6 ++---- tests/keras_contrib/callbacks/dead_relu_detector_test.py | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/keras_contrib/callbacks/dead_relu_detector.py b/keras_contrib/callbacks/dead_relu_detector.py index 868fc16..f545ee2 100644 --- a/keras_contrib/callbacks/dead_relu_detector.py +++ b/keras_contrib/callbacks/dead_relu_detector.py @@ -63,14 +63,12 @@ class DeadReluDetector(Callback): act_len = len(shape_act) # should work for both Conv and Flat - if K.backend() == 'tensorflow': + if K.image_data_format() == 'channels_last': # features in last axis axis_filter = -1 - elif K.backend() == 'theano': + else: # features before the convolution axis, for weight_len the input and output have to be subtracted axis_filter = -1 - (weight_len - 2) - else: - raise ValueError('Unknown backend: {}'.format(K.backend())) total_featuremaps = shape_act[axis_filter] diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index 89c6bbe..552dbd0 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -96,12 +96,10 @@ def test_DeadDeadReluDetector_conv(): n_samples = 9 # (5, 5) kernel, 4 input featuremaps and 10 output featuremaps - if K.backend() == 'tensorflow': + if K.image_data_format() == 'channels_last': input_shape = (n_samples, 5, 5, 4) - elif K.backend() == 'theano': - input_shape = (n_samples, 4, 5, 5) else: - raise ValueError('Unknown backend: {}'.format(K.backend())) + input_shape = (n_samples, 4, 5, 5) # ignore batch size input_shape_conv = tuple(input_shape[1:]) From 3f206044c8e2f0d21451e777a1badb760dc10e58 Mon Sep 17 00:00:00 2001 From: lameeus Date: Fri, 27 Oct 2017 11:25:07 +0200 Subject: [PATCH 05/16] changed warning method to be adjustable assert replaced by raise Error --- keras_contrib/callbacks/dead_relu_detector.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/keras_contrib/callbacks/dead_relu_detector.py b/keras_contrib/callbacks/dead_relu_detector.py index f545ee2..16ea7cc 100644 --- a/keras_contrib/callbacks/dead_relu_detector.py +++ b/keras_contrib/callbacks/dead_relu_detector.py @@ -15,13 +15,17 @@ class DeadReluDetector(Callback): True means that even a single dead neuron triggers warning False means that only significant number of dead neurons (10% or more) triggers warning + bool_warning: output mode + True means a warning is raised + False means the warning message is printed. """ - def __init__(self, x_train, verbose=False): + def __init__(self, x_train, verbose=False, bool_warning = False): super(DeadReluDetector, self).__init__() self.x_train = x_train self.verbose = verbose self.dead_neurons_share_threshold = 0.1 + self.bool_warning = bool_warning @staticmethod def is_relu_layer(layer): @@ -49,7 +53,9 @@ class DeadReluDetector(Callback): # layer_weight is a list [W] (+ [b]) layer_weight = self.model.layers[layer_index].get_weights() # with kernel and bias, the weights are saved as a list [W, b]. If only weights, it is [W] - assert type(layer_weight) == list + if type(layer_weight) is not list: + raise ValueError("'Layer_weight' should be a list, but was {}".format(type(layer_weight))) + layer_weight_shape = np.shape(layer_weight[0]) yield [layer_index, layer_activations, layer_name, layer_weight_shape] @@ -81,5 +87,9 @@ class DeadReluDetector(Callback): if (self.verbose and dead_neurons > 0) or dead_neurons_share > self.dead_neurons_share_threshold: str_warning = 'Layer {} (#{}) has {} dead neurons ({:.2%})!'.format(layer_name, layer_index, dead_neurons, dead_neurons_share) - print(str_warning) - warnings.warn(str_warning, RuntimeWarning) + + if self.bool_warning: + warnings.warn(str_warning, RuntimeWarning) + else: + print(str_warning) + From bbb08f42cbadb7e46962f5de0818bf1d420f30e3 Mon Sep 17 00:00:00 2001 From: lameeus Date: Fri, 27 Oct 2017 11:45:43 +0200 Subject: [PATCH 06/16] Test and PEP8 fixes --- keras_contrib/callbacks/dead_relu_detector.py | 5 ++--- tests/keras_contrib/callbacks/dead_relu_detector_test.py | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/keras_contrib/callbacks/dead_relu_detector.py b/keras_contrib/callbacks/dead_relu_detector.py index 16ea7cc..fedb43f 100644 --- a/keras_contrib/callbacks/dead_relu_detector.py +++ b/keras_contrib/callbacks/dead_relu_detector.py @@ -20,7 +20,7 @@ class DeadReluDetector(Callback): False means the warning message is printed. """ - def __init__(self, x_train, verbose=False, bool_warning = False): + def __init__(self, x_train, verbose=False, bool_warning=False): super(DeadReluDetector, self).__init__() self.x_train = x_train self.verbose = verbose @@ -87,9 +87,8 @@ class DeadReluDetector(Callback): if (self.verbose and dead_neurons > 0) or dead_neurons_share > self.dead_neurons_share_threshold: str_warning = 'Layer {} (#{}) has {} dead neurons ({:.2%})!'.format(layer_name, layer_index, dead_neurons, dead_neurons_share) - + if self.bool_warning: warnings.warn(str_warning, RuntimeWarning) else: print(str_warning) - diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index 552dbd0..edb5383 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -29,7 +29,7 @@ def test_DeadDeadReluDetector(): dataset, np.ones(shape_out), epochs=1, - callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], + callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose, bool_warning=True)], verbose=False ) assert len(w) == expected_warnings @@ -71,7 +71,7 @@ def test_DeadDeadReluDetector_bias(): dataset, np.ones(shape_out), epochs=1, - callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], + callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose, bool_warning=True)], verbose=False ) assert len(w) == expected_warnings @@ -119,7 +119,7 @@ def test_DeadDeadReluDetector_conv(): dataset, np.ones(shape_out), epochs=1, - callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], + callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose, bool_warning=True)], verbose=False ) assert len(w) == expected_warnings From e873325d0b59a8f7c983613af0f00f7bc99284ae Mon Sep 17 00:00:00 2001 From: lameeus Date: Mon, 6 Nov 2017 17:50:59 +0100 Subject: [PATCH 07/16] 1) Extended layers to --- keras_contrib/callbacks/dead_relu_detector.py | 18 +-- .../callbacks/dead_relu_detector_test.py | 126 +++++++++++------- 2 files changed, 82 insertions(+), 62 deletions(-) diff --git a/keras_contrib/callbacks/dead_relu_detector.py b/keras_contrib/callbacks/dead_relu_detector.py index fedb43f..f8f6839 100644 --- a/keras_contrib/callbacks/dead_relu_detector.py +++ b/keras_contrib/callbacks/dead_relu_detector.py @@ -1,5 +1,4 @@ import numpy as np -import warnings from keras.callbacks import Callback from keras import backend as K @@ -12,20 +11,16 @@ class DeadReluDetector(Callback): # Arguments x_train: Training dataset to check whether or not neurons fire verbose: verbosity mode - True means that even a single dead neuron triggers warning + True means that even a single dead neuron triggers a warning message False means that only significant number of dead neurons (10% or more) - triggers warning - bool_warning: output mode - True means a warning is raised - False means the warning message is printed. + triggers a warning message """ - def __init__(self, x_train, verbose=False, bool_warning=False): + def __init__(self, x_train, verbose=False): super(DeadReluDetector, self).__init__() self.x_train = x_train self.verbose = verbose self.dead_neurons_share_threshold = 0.1 - self.bool_warning = bool_warning @staticmethod def is_relu_layer(layer): @@ -84,11 +79,8 @@ class DeadReluDetector(Callback): dead_neurons = np.sum(np.sum(activation_values, axis=axis) == 0) dead_neurons_share = dead_neurons / total_featuremaps - if (self.verbose and dead_neurons > 0) or dead_neurons_share > self.dead_neurons_share_threshold: + if (self.verbose and dead_neurons > 0) or dead_neurons_share >= self.dead_neurons_share_threshold: str_warning = 'Layer {} (#{}) has {} dead neurons ({:.2%})!'.format(layer_name, layer_index, dead_neurons, dead_neurons_share) - if self.bool_warning: - warnings.warn(str_warning, RuntimeWarning) - else: - print(str_warning) + print(str_warning) diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index edb5383..09e6403 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -1,101 +1,124 @@ import pytest -import warnings import numpy as np +import sys +import io from keras_contrib import callbacks from keras.models import Sequential from keras.layers import Dense, Conv2D, Flatten from keras import backend as K +n_out = 11 # with 1 neuron dead, 1/11 is just below the threshold of 10% with verbose = False + + +def check_print(do_train, expected_warnings, nr_dead: int = None, perc_dead: float = None): + """ + :param perc_dead: as float, 10% should be written as 0.1 + Receive stdout to check if correct warning message is delivered. + """ + saved_stdout = sys.stdout + out = io.StringIO() + sys.stdout = out # overwrite current stdout + + do_train() + + stdoutput = out.getvalue() # get prints, can be something like: "Layer dense (#0) has 2 dead neurons (20.00%)!" + sys.stdout = saved_stdout # restore stdout + + str_count = "dead neurons" + count = stdoutput.count(str_count) + assert expected_warnings == count + if expected_warnings and (nr_dead is not None): + assert 'has {} dead'.format(nr_dead) in stdoutput + if expected_warnings and (perc_dead is not None): + assert 'neurons ({:.2%})'.format(perc_dead) in stdoutput + def test_DeadDeadReluDetector(): n_samples = 9 input_shape = (n_samples, 3, 4) # 4 input features - shape_out = (n_samples, 3, 10) # 10 output features - shape_weights = (4, 10) + shape_out = (n_samples, 3, n_out) # 11 output features + shape_weights = (4, n_out) # ignore batch size input_shape_dense = tuple(input_shape[1:]) - def do_test(weights, expected_warnings, verbose): - with warnings.catch_warnings(record=True) as w: + def do_test(weights, expected_warnings, verbose, nr_dead=None, perc_dead=None): + + def do_train(): dataset = np.ones(input_shape) # data to be fed as training model = Sequential() - model.add(Dense(10, activation='relu', input_shape=input_shape_dense, + model.add(Dense(n_out, activation='relu', input_shape=input_shape_dense, use_bias=False, weights=[weights], name='dense')) model.compile(optimizer='sgd', loss='categorical_crossentropy') model.fit( dataset, np.ones(shape_out), epochs=1, - callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose, bool_warning=True)], + callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], verbose=False ) - assert len(w) == expected_warnings - for warn_item in w: - assert issubclass(warn_item.category, RuntimeWarning) - assert "dead neurons" in str(warn_item.message) - weights_1_dead = np.ones(shape_weights) # weights that correspond to NN with 1/10 neurons dead - weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/10 neurons dead + check_print(do_train, expected_warnings, nr_dead, perc_dead) + + weights_1_dead = np.ones(shape_weights) # weights that correspond to NN with 1/11 neurons dead + weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/11 neurons dead weights_1_dead[:, 0] = 0 weights_2_dead[:, 0:2] = 0 - do_test(weights_1_dead, verbose=True, expected_warnings=1) + do_test(weights_1_dead, verbose=True, expected_warnings=1, nr_dead=1, perc_dead=1. / n_out) do_test(weights_1_dead, verbose=False, expected_warnings=0) - do_test(weights_2_dead, verbose=True, expected_warnings=1) + do_test(weights_2_dead, verbose=True, expected_warnings=1, nr_dead=2, perc_dead=2. / n_out) def test_DeadDeadReluDetector_bias(): n_samples = 9 input_shape = (n_samples, 4) # 4 input features - shape_weights = (4, 10) - shape_out = (n_samples, 10) # 10 output features - shape_bias = (10, ) + shape_weights = (4, n_out) + shape_bias = (n_out, ) + shape_out = (n_samples, n_out) # 11 output features # ignore batch size input_shape_dense = tuple(input_shape[1:]) - def do_test(weights, bias, expected_warnings, verbose): - with warnings.catch_warnings(record=True) as w: + def do_test(weights, bias, expected_warnings, verbose, nr_dead=None, perc_dead=None): + + def do_train(): dataset = np.ones(input_shape) # data to be fed as training model = Sequential() - model.add(Dense(10, activation='relu', input_shape=input_shape_dense, + model.add(Dense(n_out, activation='relu', input_shape=input_shape_dense, use_bias=True, weights=[weights, bias], name='dense')) model.compile(optimizer='sgd', loss='categorical_crossentropy') - # model.compile(optimizer=None, loss='categorical_crossentropy') model.fit( dataset, np.ones(shape_out), epochs=1, - callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose, bool_warning=True)], + callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], verbose=False ) - assert len(w) == expected_warnings - for warn_item in w: - assert issubclass(warn_item.category, RuntimeWarning) - assert "dead neurons" in str(warn_item.message) - weights_1_dead = np.ones(shape_weights) # weights that correspond to NN with 1/10 neurons dead - weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/10 neurons dead + check_print(do_train, expected_warnings, nr_dead, perc_dead) + + weights_1_dead = np.ones(shape_weights) # weights that correspond to NN with 1/11 neurons dead + weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/11 neurons dead weights_1_dead[:, 0] = 0 weights_2_dead[:, 0:2] = 0 bias = np.zeros(shape_bias) - do_test(weights_1_dead, bias, verbose=True, expected_warnings=1) + do_test(weights_1_dead, bias, verbose=True, expected_warnings=1, nr_dead=1, perc_dead=1. / n_out) do_test(weights_1_dead, bias, verbose=False, expected_warnings=0) - do_test(weights_2_dead, bias, verbose=True, expected_warnings=1) + do_test(weights_2_dead, bias, verbose=True, expected_warnings=1, nr_dead=2, perc_dead=2. / n_out) def test_DeadDeadReluDetector_conv(): n_samples = 9 - # (5, 5) kernel, 4 input featuremaps and 10 output featuremaps + # (5, 5) kernel, 4 input featuremaps and 11 output featuremaps if K.image_data_format() == 'channels_last': input_shape = (n_samples, 5, 5, 4) else: @@ -103,43 +126,48 @@ def test_DeadDeadReluDetector_conv(): # ignore batch size input_shape_conv = tuple(input_shape[1:]) - shape_weights = (5, 5, 4, 10) - shape_out = (n_samples, 10) + shape_weights = (5, 5, 4, n_out) + shape_out = (n_samples, n_out) - def do_test(weights_bias, expected_warnings, verbose): - with warnings.catch_warnings(record=True) as w: + def do_test(weights_bias, expected_warnings, verbose, nr_dead: int = None, perc_dead: float = None): + """ + :param perc_dead: as float, 10% should be written as 0.1 + """ - dataset = np.ones(input_shape) # data to be fed as training + def do_train(): + dataset = np.ones(input_shape) # data to be fed as training model = Sequential() - model.add(Conv2D(10, (5, 5), activation='relu', input_shape=input_shape_conv, + model.add(Conv2D(n_out, (5, 5), activation='relu', input_shape=input_shape_conv, use_bias=True, weights=weights_bias, name='conv')) - model.add(Flatten()) # to handle Theano's categorical crossentropy + model.add(Flatten()) # to handle Theano's categorical crossentropy model.compile(optimizer='sgd', loss='categorical_crossentropy') model.fit( dataset, np.ones(shape_out), epochs=1, - callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose, bool_warning=True)], + callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], verbose=False ) - assert len(w) == expected_warnings - for warn_item in w: - assert issubclass(warn_item.category, RuntimeWarning) - assert "dead neurons" in str(warn_item.message) - weights_1_dead = np.ones(shape_weights) # weights that correspond to NN with 1/10 neurons dead + check_print(do_train, expected_warnings, nr_dead, perc_dead) + + weights_1_dead = np.ones(shape_weights) # weights that correspond to NN with 1/11 neurons dead weights_1_dead[..., 0] = 0 - weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/10 neurons dead + weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/11 neurons dead weights_2_dead[..., 0:2] = 0 + weights_all_dead = np.ones(shape_weights) # weights that correspond to NN with all neurons dead + weights_all_dead[..., :] = 0 - bias = np.zeros((10, )) + bias = np.zeros((11, )) weights_bias_1_dead = [weights_1_dead, bias] weights_bias_2_dead = [weights_2_dead, bias] + weights_bias_all_dead = [weights_all_dead, bias] - do_test(weights_bias_1_dead, verbose=True, expected_warnings=1) + do_test(weights_bias_1_dead, verbose=True, expected_warnings=1, nr_dead=1, perc_dead=1. / n_out) do_test(weights_bias_1_dead, verbose=False, expected_warnings=0) - do_test(weights_bias_2_dead, verbose=True, expected_warnings=1) + do_test(weights_bias_2_dead, verbose=True, expected_warnings=1, nr_dead=2, perc_dead=2. / n_out) + do_test(weights_bias_all_dead, verbose=True, expected_warnings=1, nr_dead=11, perc_dead=1.) if __name__ == '__main__': From 9238838aaeb7ce08b5f6c967fc49eec0f8fdcb55 Mon Sep 17 00:00:00 2001 From: lameeus Date: Mon, 6 Nov 2017 17:52:47 +0100 Subject: [PATCH 08/16] As discussed. The 'warning' is now just a print. Test is adjusted to handle printing instead of throwing a warning. --- tests/keras_contrib/callbacks/dead_relu_detector_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index 09e6403..537f0a3 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -13,8 +13,8 @@ n_out = 11 # with 1 neuron dead, 1/11 is just below the threshold of 10% with v def check_print(do_train, expected_warnings, nr_dead: int = None, perc_dead: float = None): """ + Receive stdout to check if correct warning message is delivered :param perc_dead: as float, 10% should be written as 0.1 - Receive stdout to check if correct warning message is delivered. """ saved_stdout = sys.stdout out = io.StringIO() From 148a4f9de650495111e961bf4958bf9e439ecfa1 Mon Sep 17 00:00:00 2001 From: lameeus Date: Mon, 6 Nov 2017 18:13:46 +0100 Subject: [PATCH 09/16] Making test compatible with python 2 and hoping an unexpected bug is fixed (didn't get it at my system) --- .../callbacks/dead_relu_detector_test.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index 537f0a3..c992a8a 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -11,10 +11,11 @@ from keras import backend as K n_out = 11 # with 1 neuron dead, 1/11 is just below the threshold of 10% with verbose = False -def check_print(do_train, expected_warnings, nr_dead: int = None, perc_dead: float = None): +def check_print(do_train, expected_warnings, nr_dead=None, perc_dead=None): """ Receive stdout to check if correct warning message is delivered - :param perc_dead: as float, 10% should be written as 0.1 + :param nr_dead: int + :param perc_dead: float, 10% should be written as 0.1 """ saved_stdout = sys.stdout out = io.StringIO() @@ -23,16 +24,17 @@ def check_print(do_train, expected_warnings, nr_dead: int = None, perc_dead: flo do_train() stdoutput = out.getvalue() # get prints, can be something like: "Layer dense (#0) has 2 dead neurons (20.00%)!" + str_to_count = "dead neurons" + count = stdoutput.count(str_to_count) + sys.stdout = saved_stdout # restore stdout - - str_count = "dead neurons" - count = stdoutput.count(str_count) + assert expected_warnings == count if expected_warnings and (nr_dead is not None): assert 'has {} dead'.format(nr_dead) in stdoutput if expected_warnings and (perc_dead is not None): assert 'neurons ({:.2%})'.format(perc_dead) in stdoutput - + def test_DeadDeadReluDetector(): n_samples = 9 @@ -167,7 +169,7 @@ def test_DeadDeadReluDetector_conv(): do_test(weights_bias_1_dead, verbose=True, expected_warnings=1, nr_dead=1, perc_dead=1. / n_out) do_test(weights_bias_1_dead, verbose=False, expected_warnings=0) do_test(weights_bias_2_dead, verbose=True, expected_warnings=1, nr_dead=2, perc_dead=2. / n_out) - do_test(weights_bias_all_dead, verbose=True, expected_warnings=1, nr_dead=11, perc_dead=1.) + do_test(weights_bias_all_dead, verbose=True, expected_warnings=1, nr_dead=n_out, perc_dead=1.) if __name__ == '__main__': From 46f7a4604914546fedcb8f9bb4cc72165b7760c9 Mon Sep 17 00:00:00 2001 From: lameeus Date: Mon, 6 Nov 2017 18:23:13 +0100 Subject: [PATCH 10/16] Made compatible with python 2.7 and tried to fix weird bug I didn't encounter at own system --- .../callbacks/dead_relu_detector_test.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index c992a8a..9cc38af 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -26,15 +26,17 @@ def check_print(do_train, expected_warnings, nr_dead=None, perc_dead=None): stdoutput = out.getvalue() # get prints, can be something like: "Layer dense (#0) has 2 dead neurons (20.00%)!" str_to_count = "dead neurons" count = stdoutput.count(str_to_count) - + sys.stdout = saved_stdout # restore stdout - + assert expected_warnings == count if expected_warnings and (nr_dead is not None): - assert 'has {} dead'.format(nr_dead) in stdoutput + str_to_check = 'has {} dead'.format(nr_dead) + assert str_to_check in stdoutput, '"{}" not in "{}"!'.format(str_to_check, stdoutput) if expected_warnings and (perc_dead is not None): - assert 'neurons ({:.2%})'.format(perc_dead) in stdoutput - + str_to_check = 'neurons ({:.2%})'.format(perc_dead) + assert str_to_check in stdoutput, '"{}" not in "{}"!'.format(str_to_check, stdoutput) + def test_DeadDeadReluDetector(): n_samples = 9 @@ -173,4 +175,5 @@ def test_DeadDeadReluDetector_conv(): if __name__ == '__main__': - pytest.main([__file__]) + # pytest.main([__file__]) + test_DeadDeadReluDetector_conv() From b2520b935ddef481499966c6ac4ecde28264601d Mon Sep 17 00:00:00 2001 From: lameeus Date: Mon, 6 Nov 2017 18:23:43 +0100 Subject: [PATCH 11/16] forgot to remove comment --- tests/keras_contrib/callbacks/dead_relu_detector_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index 9cc38af..134be70 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -175,5 +175,4 @@ def test_DeadDeadReluDetector_conv(): if __name__ == '__main__': - # pytest.main([__file__]) test_DeadDeadReluDetector_conv() From f975a52ea91334d0430d6e1be0a03c2e80a8a408 Mon Sep 17 00:00:00 2001 From: lameeus Date: Mon, 6 Nov 2017 18:47:22 +0100 Subject: [PATCH 12/16] Second try. For some reason it doesn't print with all weights 0 at server side. --- tests/keras_contrib/callbacks/dead_relu_detector_test.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index 134be70..84f1b91 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -133,7 +133,7 @@ def test_DeadDeadReluDetector_conv(): shape_weights = (5, 5, 4, n_out) shape_out = (n_samples, n_out) - def do_test(weights_bias, expected_warnings, verbose, nr_dead: int = None, perc_dead: float = None): + def do_test(weights_bias, expected_warnings, verbose, nr_dead = None, perc_dead = None): """ :param perc_dead: as float, 10% should be written as 0.1 """ @@ -159,8 +159,7 @@ def test_DeadDeadReluDetector_conv(): weights_1_dead[..., 0] = 0 weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/11 neurons dead weights_2_dead[..., 0:2] = 0 - weights_all_dead = np.ones(shape_weights) # weights that correspond to NN with all neurons dead - weights_all_dead[..., :] = 0 + weights_all_dead = np.zeros(shape_weights) # weights that correspond to NN with all neurons dead bias = np.zeros((11, )) @@ -171,8 +170,8 @@ def test_DeadDeadReluDetector_conv(): do_test(weights_bias_1_dead, verbose=True, expected_warnings=1, nr_dead=1, perc_dead=1. / n_out) do_test(weights_bias_1_dead, verbose=False, expected_warnings=0) do_test(weights_bias_2_dead, verbose=True, expected_warnings=1, nr_dead=2, perc_dead=2. / n_out) - do_test(weights_bias_all_dead, verbose=True, expected_warnings=1, nr_dead=n_out, perc_dead=1.) + do_test(weights_bias_all_dead, verbose=True, expected_warnings=1, nr_dead=n_out, perc_dead=n_out / n_out) if __name__ == '__main__': - test_DeadDeadReluDetector_conv() + pytest.main([__file__]) From 9af7625e0f704766950c925eed14379e47748f76 Mon Sep 17 00:00:00 2001 From: lameeus Date: Mon, 6 Nov 2017 19:26:50 +0100 Subject: [PATCH 13/16] fixed python 2 bug with divison StringIO depends now on python 2/3 as it should PEP8 fixes --- keras_contrib/callbacks/dead_relu_detector.py | 2 +- .../callbacks/dead_relu_detector_test.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/keras_contrib/callbacks/dead_relu_detector.py b/keras_contrib/callbacks/dead_relu_detector.py index f8f6839..2cfe37b 100644 --- a/keras_contrib/callbacks/dead_relu_detector.py +++ b/keras_contrib/callbacks/dead_relu_detector.py @@ -78,7 +78,7 @@ class DeadReluDetector(Callback): dead_neurons = np.sum(np.sum(activation_values, axis=axis) == 0) - dead_neurons_share = dead_neurons / total_featuremaps + dead_neurons_share = float(dead_neurons) / float(total_featuremaps) if (self.verbose and dead_neurons > 0) or dead_neurons_share >= self.dead_neurons_share_threshold: str_warning = 'Layer {} (#{}) has {} dead neurons ({:.2%})!'.format(layer_name, layer_index, dead_neurons, dead_neurons_share) diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index 84f1b91..e4a1783 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -1,7 +1,11 @@ import pytest import numpy as np import sys -import io + +if (sys.version_info > (3, 0)): + from io import StringIO +else: + from StringIO import StringIO from keras_contrib import callbacks from keras.models import Sequential @@ -17,8 +21,10 @@ def check_print(do_train, expected_warnings, nr_dead=None, perc_dead=None): :param nr_dead: int :param perc_dead: float, 10% should be written as 0.1 """ + saved_stdout = sys.stdout - out = io.StringIO() + + out = StringIO() sys.stdout = out # overwrite current stdout do_train() @@ -133,7 +139,7 @@ def test_DeadDeadReluDetector_conv(): shape_weights = (5, 5, 4, n_out) shape_out = (n_samples, n_out) - def do_test(weights_bias, expected_warnings, verbose, nr_dead = None, perc_dead = None): + def do_test(weights_bias, expected_warnings, verbose, nr_dead=None, perc_dead=None): """ :param perc_dead: as float, 10% should be written as 0.1 """ From 58d1817d4cf6797ceccc9528f663c714132985a5 Mon Sep 17 00:00:00 2001 From: lameeus Date: Mon, 6 Nov 2017 19:29:00 +0100 Subject: [PATCH 14/16] PEP8 --- tests/keras_contrib/callbacks/dead_relu_detector_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index e4a1783..4026890 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -21,7 +21,7 @@ def check_print(do_train, expected_warnings, nr_dead=None, perc_dead=None): :param nr_dead: int :param perc_dead: float, 10% should be written as 0.1 """ - + saved_stdout = sys.stdout out = StringIO() From a1079c012a69171cef501c39704303adfdafb8a9 Mon Sep 17 00:00:00 2001 From: lameeus Date: Tue, 7 Nov 2017 11:10:39 +0100 Subject: [PATCH 15/16] I still can't reproduce the bug I get at server side. I commented those lines of code. If someone else could find out what is going wrong? --- .../callbacks/dead_relu_detector_test.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index 4026890..5f7c396 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -25,23 +25,25 @@ def check_print(do_train, expected_warnings, nr_dead=None, perc_dead=None): saved_stdout = sys.stdout out = StringIO() + out.flush() sys.stdout = out # overwrite current stdout do_train() - stdoutput = out.getvalue() # get prints, can be something like: "Layer dense (#0) has 2 dead neurons (20.00%)!" + stdoutput = out.getvalue().strip() # get prints, can be something like: "Layer dense (#0) has 2 dead neurons (20.00%)!" str_to_count = "dead neurons" count = stdoutput.count(str_to_count) sys.stdout = saved_stdout # restore stdout + out.close() assert expected_warnings == count if expected_warnings and (nr_dead is not None): str_to_check = 'has {} dead'.format(nr_dead) - assert str_to_check in stdoutput, '"{}" not in "{}"!'.format(str_to_check, stdoutput) + assert str_to_check in stdoutput, '"{}" not in "{}"'.format(str_to_check, stdoutput) if expected_warnings and (perc_dead is not None): - str_to_check = 'neurons ({:.2%})'.format(perc_dead) - assert str_to_check in stdoutput, '"{}" not in "{}"!'.format(str_to_check, stdoutput) + str_to_check = 'neurons ({:.2%})!'.format(perc_dead) + assert str_to_check in stdoutput, '"{}" not in "{}"'.format(str_to_check, stdoutput) def test_DeadDeadReluDetector(): @@ -65,6 +67,7 @@ def test_DeadDeadReluDetector(): model.fit( dataset, np.ones(shape_out), + batch_size=1, epochs=1, callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], verbose=False @@ -74,6 +77,7 @@ def test_DeadDeadReluDetector(): weights_1_dead = np.ones(shape_weights) # weights that correspond to NN with 1/11 neurons dead weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/11 neurons dead + weights_all_dead = np.zeros(shape_weights) # weights that correspond to all neurons dead weights_1_dead[:, 0] = 0 weights_2_dead[:, 0:2] = 0 @@ -81,6 +85,7 @@ def test_DeadDeadReluDetector(): do_test(weights_1_dead, verbose=True, expected_warnings=1, nr_dead=1, perc_dead=1. / n_out) do_test(weights_1_dead, verbose=False, expected_warnings=0) do_test(weights_2_dead, verbose=True, expected_warnings=1, nr_dead=2, perc_dead=2. / n_out) + # do_test(weights_all_dead, verbose=True, expected_warnings=1, nr_dead=n_out, perc_dead=1.) def test_DeadDeadReluDetector_bias(): @@ -105,6 +110,7 @@ def test_DeadDeadReluDetector_bias(): model.fit( dataset, np.ones(shape_out), + batch_size=1, epochs=1, callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], verbose=False @@ -114,6 +120,7 @@ def test_DeadDeadReluDetector_bias(): weights_1_dead = np.ones(shape_weights) # weights that correspond to NN with 1/11 neurons dead weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/11 neurons dead + weights_all_dead = np.zeros(shape_weights) # weights that correspond to all neurons dead weights_1_dead[:, 0] = 0 weights_2_dead[:, 0:2] = 0 @@ -123,6 +130,7 @@ def test_DeadDeadReluDetector_bias(): do_test(weights_1_dead, bias, verbose=True, expected_warnings=1, nr_dead=1, perc_dead=1. / n_out) do_test(weights_1_dead, bias, verbose=False, expected_warnings=0) do_test(weights_2_dead, bias, verbose=True, expected_warnings=1, nr_dead=2, perc_dead=2. / n_out) + # do_test(weights_all_dead, bias, verbose=True, expected_warnings=1, nr_dead=n_out, perc_dead=1.) def test_DeadDeadReluDetector_conv(): @@ -154,6 +162,7 @@ def test_DeadDeadReluDetector_conv(): model.fit( dataset, np.ones(shape_out), + batch_size=1, epochs=1, callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], verbose=False @@ -176,7 +185,7 @@ def test_DeadDeadReluDetector_conv(): do_test(weights_bias_1_dead, verbose=True, expected_warnings=1, nr_dead=1, perc_dead=1. / n_out) do_test(weights_bias_1_dead, verbose=False, expected_warnings=0) do_test(weights_bias_2_dead, verbose=True, expected_warnings=1, nr_dead=2, perc_dead=2. / n_out) - do_test(weights_bias_all_dead, verbose=True, expected_warnings=1, nr_dead=n_out, perc_dead=n_out / n_out) + # do_test(weights_bias_all_dead, verbose=True, expected_warnings=1, nr_dead=n_out, perc_dead=1.) if __name__ == '__main__': From 68acf5e2eea5c8dac90f38c1e7782456ec61faa6 Mon Sep 17 00:00:00 2001 From: lameeus Date: Mon, 20 Nov 2017 11:46:24 +0100 Subject: [PATCH 16/16] added absolute tolerance to handle comparing zeros --- tests/keras_contrib/backend/backend_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/keras_contrib/backend/backend_test.py b/tests/keras_contrib/backend/backend_test.py index 998bc6b..1e4f344 100644 --- a/tests/keras_contrib/backend/backend_test.py +++ b/tests/keras_contrib/backend/backend_test.py @@ -157,8 +157,9 @@ class TestBackend(object): th_var_val = KTH.eval(th_var) tf_var_val = KTF.eval(tf_var) - assert_allclose(th_mean_val, tf_mean_val, rtol=1e-4) - assert_allclose(th_var_val, tf_var_val, rtol=1e-4) + # absolute tolerance needed when working with zeros + assert_allclose(th_mean_val, tf_mean_val, rtol=1e-4, atol=1e-10) + assert_allclose(th_var_val, tf_var_val, rtol=1e-4, atol=1e-10) def test_clip(self): check_single_tensor_operation('clip', (4, 2), min_value=0.4, max_value=0.6)