From 0be5a2ccac4e7ae3e0cff578499e5349eca100ec Mon Sep 17 00:00:00 2001 From: vfdev-5 Date: Sun, 1 Oct 2017 01:52:23 +0200 Subject: [PATCH 01/29] * Add names for layers of dense/transition blocks --- keras_contrib/applications/densenet.py | 82 ++++++++++++++++---------- 1 file changed, 50 insertions(+), 32 deletions(-) diff --git a/keras_contrib/applications/densenet.py b/keras_contrib/applications/densenet.py index 290d9cc..b885703 100644 --- a/keras_contrib/applications/densenet.py +++ b/keras_contrib/applications/densenet.py @@ -506,7 +506,11 @@ def DenseNetImageNet161(input_shape=None, pooling=pooling, classes=classes, activation=activation) -def __conv_block(ip, nb_filter, bottleneck=False, dropout_rate=None, weight_decay=1e-4): +def name_or_none(prefix, name): + return prefix + name if (prefix is not None and name is not None) else None + + +def __conv_block(ip, nb_filter, bottleneck=False, dropout_rate=None, weight_decay=1e-4, block_prefix=None): ''' Adds a convolution layer (with batch normalization and relu), and optionally a bottleneck layer. @@ -518,6 +522,7 @@ def __conv_block(ip, nb_filter, bottleneck=False, dropout_rate=None, weight_deca bottleneck: if True, adds a bottleneck convolution block dropout_rate: dropout rate weight_decay: weight decay factor + block_prefix: str, for unique layer naming # Input shape 4D tensor with shape: @@ -538,18 +543,20 @@ def __conv_block(ip, nb_filter, bottleneck=False, dropout_rate=None, weight_deca with K.name_scope('ConvBlock'): concat_axis = 1 if K.image_data_format() == 'channels_first' else -1 - x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(ip) + x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5, name=name_or_none(block_prefix, '_bn'))(ip) x = Activation('relu')(x) if bottleneck: inter_channel = nb_filter * 4 x = Conv2D(inter_channel, (1, 1), kernel_initializer='he_normal', padding='same', use_bias=False, - kernel_regularizer=l2(weight_decay))(x) - x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(x) + kernel_regularizer=l2(weight_decay), name=name_or_none(block_prefix, '_bottleneck_conv2D'))(x) + x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5, + name=name_or_none(block_prefix, '_bottleneck_bn'))(x) x = Activation('relu')(x) - x = Conv2D(nb_filter, (3, 3), kernel_initializer='he_normal', padding='same', use_bias=False)(x) + x = Conv2D(nb_filter, (3, 3), kernel_initializer='he_normal', padding='same', use_bias=False, + name=name_or_none(block_prefix, '_conv2D'))(x) if dropout_rate: x = Dropout(dropout_rate)(x) @@ -557,7 +564,7 @@ def __conv_block(ip, nb_filter, bottleneck=False, dropout_rate=None, weight_deca def __dense_block(x, nb_layers, nb_filter, growth_rate, bottleneck=False, dropout_rate=None, - weight_decay=1e-4, grow_nb_filters=True, return_concat_list=False): + weight_decay=1e-4, grow_nb_filters=True, return_concat_list=False, block_prefix=None): ''' Build a dense_block where the output of each conv_block is fed to subsequent ones @@ -575,6 +582,7 @@ def __dense_block(x, nb_layers, nb_filter, growth_rate, bottleneck=False, dropou grow_nb_filters: if True, allows number of filters to grow return_concat_list: set to True to return the list of feature maps along with the actual output + block_prefix: str, for block unique naming # Return If return_concat_list is True, returns a list of the output @@ -590,7 +598,8 @@ def __dense_block(x, nb_layers, nb_filter, growth_rate, bottleneck=False, dropou x_list = [x] for i in range(nb_layers): - cb = __conv_block(x, growth_rate, bottleneck, dropout_rate, weight_decay) + cb = __conv_block(x, growth_rate, bottleneck, dropout_rate, weight_decay, + block_prefix=name_or_none(block_prefix, '_%i' % i)) x_list.append(cb) x = concatenate([x, cb], axis=concat_axis) @@ -604,7 +613,7 @@ def __dense_block(x, nb_layers, nb_filter, growth_rate, bottleneck=False, dropou return x, nb_filter -def __transition_block(ip, nb_filter, compression=1.0, weight_decay=1e-4): +def __transition_block(ip, nb_filter, compression=1.0, weight_decay=1e-4, block_prefix=None): ''' Adds a pointwise convolution layer (with batch normalization and relu), and an average pooling layer. The number of output convolution filters @@ -617,6 +626,7 @@ def __transition_block(ip, nb_filter, compression=1.0, weight_decay=1e-4): compression: calculated as 1 - reduction. Reduces the number of feature maps in the transition block. weight_decay: weight decay factor + block_prefix: str, for block unique naming # Input shape 4D tensor with shape: @@ -638,16 +648,16 @@ def __transition_block(ip, nb_filter, compression=1.0, weight_decay=1e-4): with K.name_scope('Transition'): concat_axis = 1 if K.image_data_format() == 'channels_first' else -1 - x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(ip) + x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5, name=name_or_none(block_prefix, '_bn'))(ip) x = Activation('relu')(x) x = Conv2D(int(nb_filter * compression), (1, 1), kernel_initializer='he_normal', padding='same', - use_bias=False, kernel_regularizer=l2(weight_decay))(x) + use_bias=False, kernel_regularizer=l2(weight_decay), name=name_or_none(block_prefix, '_conv2D'))(x) x = AveragePooling2D((2, 2), strides=(2, 2))(x) return x -def __transition_up_block(ip, nb_filters, type='deconv', weight_decay=1E-4): +def __transition_up_block(ip, nb_filters, type='deconv', weight_decay=1E-4, block_prefix=None): '''Adds an upsampling block. Upsampling operation relies on the the type parameter. # Arguments @@ -657,6 +667,7 @@ def __transition_up_block(ip, nb_filters, type='deconv', weight_decay=1E-4): type: can be 'upsampling', 'subpixel', 'deconv'. Determines type of upsampling performed weight_decay: weight decay factor + block_prefix: str, for block unique naming # Input shape 4D tensor with shape: @@ -676,17 +687,17 @@ def __transition_up_block(ip, nb_filters, type='deconv', weight_decay=1E-4): with K.name_scope('TransitionUp'): if type == 'upsampling': - x = UpSampling2D()(ip) + x = UpSampling2D(name=name_or_none(block_prefix, '_upsampling'))(ip) elif type == 'subpixel': x = Conv2D(nb_filters, (3, 3), activation='relu', padding='same', kernel_regularizer=l2(weight_decay), - use_bias=False, kernel_initializer='he_normal')(ip) - x = SubPixelUpscaling(scale_factor=2)(x) + use_bias=False, kernel_initializer='he_normal', name=name_or_none(block_prefix, '_conv2D'))(ip) + x = SubPixelUpscaling(scale_factor=2, name=name_or_none(block_prefix, '_subpixel'))(x) x = Conv2D(nb_filters, (3, 3), activation='relu', padding='same', kernel_regularizer=l2(weight_decay), - use_bias=False, kernel_initializer='he_normal')(x) + use_bias=False, kernel_initializer='he_normal', name=name_or_none(block_prefix, '_conv2D'))(x) else: x = Conv2DTranspose(nb_filters, (3, 3), activation='relu', padding='same', strides=(2, 2), - kernel_initializer='he_normal', kernel_regularizer=l2(weight_decay))(ip) - + kernel_initializer='he_normal', kernel_regularizer=l2(weight_decay), + name=name_or_none(block_prefix, '_conv2DT'))(ip) return x @@ -781,27 +792,30 @@ def __create_dense_net(nb_classes, img_input, include_top, depth=40, nb_dense_bl initial_kernel = (3, 3) initial_strides = (1, 1) - x = Conv2D(nb_filter, initial_kernel, kernel_initializer='he_normal', padding='same', + x = Conv2D(nb_filter, initial_kernel, kernel_initializer='he_normal', padding='same', name='initial_conv2D', strides=initial_strides, use_bias=False, kernel_regularizer=l2(weight_decay))(img_input) if subsample_initial_block: - x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(x) + x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5, name='initial_bn')(x) x = Activation('relu')(x) x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x) # Add dense blocks for block_idx in range(nb_dense_block - 1): x, nb_filter = __dense_block(x, nb_layers[block_idx], nb_filter, growth_rate, bottleneck=bottleneck, - dropout_rate=dropout_rate, weight_decay=weight_decay) + dropout_rate=dropout_rate, weight_decay=weight_decay, + block_prefix='dense_%i' % block_idx) # add transition_block - x = __transition_block(x, nb_filter, compression=compression, weight_decay=weight_decay) + x = __transition_block(x, nb_filter, compression=compression, weight_decay=weight_decay, + block_prefix='tr_%i' % block_idx) nb_filter = int(nb_filter * compression) # The last dense_block does not have a transition_block x, nb_filter = __dense_block(x, final_nb_layer, nb_filter, growth_rate, bottleneck=bottleneck, - dropout_rate=dropout_rate, weight_decay=weight_decay) + dropout_rate=dropout_rate, weight_decay=weight_decay, + block_prefix='dense_%i' % (nb_dense_block - 1)) - x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(x) + x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5, name='final_bn')(x) x = Activation('relu')(x) if include_top: @@ -889,7 +903,7 @@ def __create_fcn_dense_net(nb_classes, img_input, include_top, nb_dense_block=5, # Initial convolution x = Conv2D(init_conv_filters, (7, 7), kernel_initializer='he_normal', padding='same', name='initial_conv2D', use_bias=False, kernel_regularizer=l2(weight_decay))(img_input) - x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(x) + x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5, name='initial_bn')(x) x = Activation('relu')(x) nb_filter = init_conv_filters @@ -899,13 +913,14 @@ def __create_fcn_dense_net(nb_classes, img_input, include_top, nb_dense_block=5, # Add dense blocks and transition down block for block_idx in range(nb_dense_block): x, nb_filter = __dense_block(x, nb_layers[block_idx], nb_filter, growth_rate, dropout_rate=dropout_rate, - weight_decay=weight_decay) + weight_decay=weight_decay, block_prefix='dense_%i' % block_idx) # Skip connection skip_list.append(x) # add transition_block - x = __transition_block(x, nb_filter, compression=compression, weight_decay=weight_decay) + x = __transition_block(x, nb_filter, compression=compression, weight_decay=weight_decay, + block_prefix='tr_%i' % block_idx) nb_filter = int(nb_filter * compression) # this is calculated inside transition_down_block @@ -913,7 +928,8 @@ def __create_fcn_dense_net(nb_classes, img_input, include_top, nb_dense_block=5, # return the concatenated feature maps without the concatenation of the input _, nb_filter, concat_list = __dense_block(x, bottleneck_nb_layers, nb_filter, growth_rate, dropout_rate=dropout_rate, weight_decay=weight_decay, - return_concat_list=True) + return_concat_list=True, + block_prefix='dense_%i' % nb_dense_block) skip_list = skip_list[::-1] # reverse the skip list @@ -925,16 +941,18 @@ def __create_fcn_dense_net(nb_classes, img_input, include_top, nb_dense_block=5, # not the concatenation of the input with the feature maps (concat_list[0]. l = concatenate(concat_list[1:], axis=concat_axis) - t = __transition_up_block(l, nb_filters=n_filters_keep, type=upsampling_type, weight_decay=weight_decay) + t = __transition_up_block(l, nb_filters=n_filters_keep, type=upsampling_type, weight_decay=weight_decay, + block_prefix='tr_up_%i' % block_idx) # concatenate the skip connection with the transition block x = concatenate([t, skip_list[block_idx]], axis=concat_axis) # Dont allow the feature map size to grow in upsampling dense blocks - x_up, nb_filter, concat_list = __dense_block(x, nb_layers[nb_dense_block + block_idx + 1], nb_filter=growth_rate, - growth_rate=growth_rate, dropout_rate=dropout_rate, - weight_decay=weight_decay, return_concat_list=True, - grow_nb_filters=False) + x_up, nb_filter, concat_list = __dense_block(x, nb_layers[nb_dense_block + block_idx + 1], + nb_filter=growth_rate, growth_rate=growth_rate, + dropout_rate=dropout_rate, weight_decay=weight_decay, + return_concat_list=True, grow_nb_filters=False, + block_prefix='dense_%i' % (nb_dense_block + 1 + block_idx)) if include_top: x = Conv2D(nb_classes, (1, 1), activation='linear', padding='same', use_bias=False)(x_up) From 64ee9acfdbef6fe519ed17decd2aebdb778e64b0 Mon Sep 17 00:00:00 2001 From: lameeus Date: Wed, 25 Oct 2017 12:13:09 +0200 Subject: [PATCH 02/29] 1) Extended layers to everything that has a ReLU activation 2) Added layer name to the warning 3) Corrected the dead neuron count (should work for both Theano and TF (tested for TF)) 4) Added a print alongside the warning since for me the warning stopped showing after some epochs (might be a bug at my side). --- keras_contrib/callbacks/dead_relu_detector.py | 54 ++++++++++++++----- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/keras_contrib/callbacks/dead_relu_detector.py b/keras_contrib/callbacks/dead_relu_detector.py index 2019f56..6ad2074 100644 --- a/keras_contrib/callbacks/dead_relu_detector.py +++ b/keras_contrib/callbacks/dead_relu_detector.py @@ -2,7 +2,6 @@ import numpy as np import warnings from keras.callbacks import Callback -from keras.layers import Dense from keras import backend as K @@ -17,6 +16,7 @@ class DeadReluDetector(Callback): False means that only significant number of dead neurons (10% or more) triggers warning """ + def __init__(self, x_train, verbose=False): super(DeadReluDetector, self).__init__() self.x_train = x_train @@ -25,7 +25,11 @@ class DeadReluDetector(Callback): @staticmethod def is_relu_layer(layer): - return isinstance(layer, Dense) and layer.get_config()['activation'] == 'relu' + # Should work for all layers with relu activation. Tested for Dense and Conv2D + if 'activation' in layer.get_config(): + return layer.get_config()['activation'] == 'relu' + else: + return False def get_relu_activations(self): model_input = self.model.input @@ -44,17 +48,43 @@ class DeadReluDetector(Callback): layer_outputs = [func(list_inputs)[0] for func in funcs] for layer_index, layer_activations in enumerate(layer_outputs): if self.is_relu_layer(self.model.layers[layer_index]): - yield [layer_index, layer_activations] + layer_name = self.model.layers[layer_index].name + # layer_weight is a list [W] (+ [b]) + layer_weight = self.model.layers[layer_index].get_weights() + # with kernel and bias, the weights are saved as a list [W, b]. If only weights, it is [W] + assert type(layer_weight) == list + layer_weight_shape = np.shape(layer_weight[0]) + yield [layer_index, layer_activations, layer_name, layer_weight_shape] def on_epoch_end(self, epoch, logs={}): for relu_activation in self.get_relu_activations(): - layer_index, activation_values = relu_activation - total_neurons = activation_values.shape[-1] - dead_neurons = np.sum(activation_values == 0) - dead_neurons_share = dead_neurons / total_neurons + layer_index, activation_values, layer_name, layer_weight_shape = relu_activation + + shape_act = activation_values.shape + + weight_len = len(layer_weight_shape) + act_len = len(shape_act) + + # should work for both Conv and Flat + if K.backend() == 'tensorflow': + # features in last axis + axis_filter = -1 + elif K.backend() == 'theano': + # features before the convolution axis, for weight_len the input and output have to be subtracted + axis_filter = -1 - (weight_len - 2) + else: + raise ValueError('Unknown backend: {}'.format(K.backend())) + + total_featuremaps = shape_act[axis_filter] + + axis = tuple( + i for i in range(act_len) if (i != axis_filter) and (i != (len(shape_act) + axis_filter))) + + dead_neurons = np.sum(np.sum(activation_values, axis=axis) == 0) + + dead_neurons_share = dead_neurons / total_featuremaps if (self.verbose and dead_neurons > 0) or dead_neurons_share > self.dead_neurons_share_threshold: - warnings.warn( - 'Layer #{} has {} dead neurons ({:.2%})!' - .format(layer_index, dead_neurons, dead_neurons_share), - RuntimeWarning - ) + str_warning = 'Layer {} (#{}) has {} dead neurons ({:.2%})!'.format(layer_name, layer_index, + dead_neurons, dead_neurons_share) + print(str_warning) + warnings.warn(str_warning, RuntimeWarning) From d635747da3cfe5deb5782401139d1942f029faeb Mon Sep 17 00:00:00 2001 From: lameeus Date: Wed, 25 Oct 2017 12:14:39 +0200 Subject: [PATCH 03/29] Made first test more general + test if bias included + test when using a convolutional layer (Conv2D) --- .../callbacks/dead_relu_detector_test.py | 123 ++++++++++++++++-- 1 file changed, 115 insertions(+), 8 deletions(-) diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index 9a37df9..89c6bbe 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -4,19 +4,30 @@ import numpy as np from keras_contrib import callbacks from keras.models import Sequential -from keras.layers import Dense +from keras.layers import Dense, Conv2D, Flatten +from keras import backend as K def test_DeadDeadReluDetector(): + n_samples = 9 + + input_shape = (n_samples, 3, 4) # 4 input features + shape_out = (n_samples, 3, 10) # 10 output features + shape_weights = (4, 10) + + # ignore batch size + input_shape_dense = tuple(input_shape[1:]) + def do_test(weights, expected_warnings, verbose): with warnings.catch_warnings(record=True) as w: - dataset = np.ones((1, 1, 1)) # data to be fed as training + dataset = np.ones(input_shape) # data to be fed as training model = Sequential() - model.add(Dense(10, activation='relu', input_shape=(1, 1), use_bias=False, weights=[weights])) + model.add(Dense(10, activation='relu', input_shape=input_shape_dense, + use_bias=False, weights=[weights], name='dense')) model.compile(optimizer='sgd', loss='categorical_crossentropy') model.fit( dataset, - np.ones((1, 1, 10)), + np.ones(shape_out), epochs=1, callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], verbose=False @@ -26,16 +37,112 @@ def test_DeadDeadReluDetector(): assert issubclass(warn_item.category, RuntimeWarning) assert "dead neurons" in str(warn_item.message) - weights_1_dead = np.ones((1, 10)) # weights that correspond to NN with 1/10 neurons dead + weights_1_dead = np.ones(shape_weights) # weights that correspond to NN with 1/10 neurons dead + weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/10 neurons dead + weights_1_dead[:, 0] = 0 - weights_2_dead = np.ones((1, 10)) # weights that correspond to NN with 2/10 neurons dead - weights_2_dead[:, 0] = 0 - weights_2_dead[:, 1] = 0 + weights_2_dead[:, 0:2] = 0 do_test(weights_1_dead, verbose=True, expected_warnings=1) do_test(weights_1_dead, verbose=False, expected_warnings=0) do_test(weights_2_dead, verbose=True, expected_warnings=1) +def test_DeadDeadReluDetector_bias(): + n_samples = 9 + + input_shape = (n_samples, 4) # 4 input features + shape_weights = (4, 10) + shape_out = (n_samples, 10) # 10 output features + shape_bias = (10, ) + + # ignore batch size + input_shape_dense = tuple(input_shape[1:]) + + def do_test(weights, bias, expected_warnings, verbose): + with warnings.catch_warnings(record=True) as w: + dataset = np.ones(input_shape) # data to be fed as training + model = Sequential() + model.add(Dense(10, activation='relu', input_shape=input_shape_dense, + use_bias=True, weights=[weights, bias], name='dense')) + model.compile(optimizer='sgd', loss='categorical_crossentropy') + # model.compile(optimizer=None, loss='categorical_crossentropy') + model.fit( + dataset, + np.ones(shape_out), + epochs=1, + callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], + verbose=False + ) + assert len(w) == expected_warnings + for warn_item in w: + assert issubclass(warn_item.category, RuntimeWarning) + assert "dead neurons" in str(warn_item.message) + + weights_1_dead = np.ones(shape_weights) # weights that correspond to NN with 1/10 neurons dead + weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/10 neurons dead + + weights_1_dead[:, 0] = 0 + weights_2_dead[:, 0:2] = 0 + + bias = np.zeros(shape_bias) + + do_test(weights_1_dead, bias, verbose=True, expected_warnings=1) + do_test(weights_1_dead, bias, verbose=False, expected_warnings=0) + do_test(weights_2_dead, bias, verbose=True, expected_warnings=1) + + +def test_DeadDeadReluDetector_conv(): + n_samples = 9 + + # (5, 5) kernel, 4 input featuremaps and 10 output featuremaps + if K.backend() == 'tensorflow': + input_shape = (n_samples, 5, 5, 4) + elif K.backend() == 'theano': + input_shape = (n_samples, 4, 5, 5) + else: + raise ValueError('Unknown backend: {}'.format(K.backend())) + + # ignore batch size + input_shape_conv = tuple(input_shape[1:]) + shape_weights = (5, 5, 4, 10) + shape_out = (n_samples, 10) + + def do_test(weights_bias, expected_warnings, verbose): + with warnings.catch_warnings(record=True) as w: + + dataset = np.ones(input_shape) # data to be fed as training + model = Sequential() + model.add(Conv2D(10, (5, 5), activation='relu', input_shape=input_shape_conv, + use_bias=True, weights=weights_bias, name='conv')) + model.add(Flatten()) # to handle Theano's categorical crossentropy + model.compile(optimizer='sgd', loss='categorical_crossentropy') + model.fit( + dataset, + np.ones(shape_out), + epochs=1, + callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], + verbose=False + ) + assert len(w) == expected_warnings + for warn_item in w: + assert issubclass(warn_item.category, RuntimeWarning) + assert "dead neurons" in str(warn_item.message) + + weights_1_dead = np.ones(shape_weights) # weights that correspond to NN with 1/10 neurons dead + weights_1_dead[..., 0] = 0 + weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/10 neurons dead + weights_2_dead[..., 0:2] = 0 + + bias = np.zeros((10, )) + + weights_bias_1_dead = [weights_1_dead, bias] + weights_bias_2_dead = [weights_2_dead, bias] + + do_test(weights_bias_1_dead, verbose=True, expected_warnings=1) + do_test(weights_bias_1_dead, verbose=False, expected_warnings=0) + do_test(weights_bias_2_dead, verbose=True, expected_warnings=1) + + if __name__ == '__main__': pytest.main([__file__]) From 3d710d4984cfd6a4462642221ad4c7ac838744ba Mon Sep 17 00:00:00 2001 From: lameeus Date: Wed, 25 Oct 2017 13:32:01 +0200 Subject: [PATCH 04/29] Simplification (suggested by @arodiss) --- keras_contrib/callbacks/dead_relu_detector.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/keras_contrib/callbacks/dead_relu_detector.py b/keras_contrib/callbacks/dead_relu_detector.py index 6ad2074..868fc16 100644 --- a/keras_contrib/callbacks/dead_relu_detector.py +++ b/keras_contrib/callbacks/dead_relu_detector.py @@ -26,10 +26,7 @@ class DeadReluDetector(Callback): @staticmethod def is_relu_layer(layer): # Should work for all layers with relu activation. Tested for Dense and Conv2D - if 'activation' in layer.get_config(): - return layer.get_config()['activation'] == 'relu' - else: - return False + return 'activation' in layer.get_config() and layer.get_config()['activation'] == 'relu' def get_relu_activations(self): model_input = self.model.input From ececa1b7afe09e3566ba2f19004aa24a655dd52f Mon Sep 17 00:00:00 2001 From: lameeus Date: Wed, 25 Oct 2017 13:45:52 +0200 Subject: [PATCH 05/29] Made indepent of backend. Now only checks channel priority (as suggested by @arodiss) --- keras_contrib/callbacks/dead_relu_detector.py | 6 ++---- tests/keras_contrib/callbacks/dead_relu_detector_test.py | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/keras_contrib/callbacks/dead_relu_detector.py b/keras_contrib/callbacks/dead_relu_detector.py index 868fc16..f545ee2 100644 --- a/keras_contrib/callbacks/dead_relu_detector.py +++ b/keras_contrib/callbacks/dead_relu_detector.py @@ -63,14 +63,12 @@ class DeadReluDetector(Callback): act_len = len(shape_act) # should work for both Conv and Flat - if K.backend() == 'tensorflow': + if K.image_data_format() == 'channels_last': # features in last axis axis_filter = -1 - elif K.backend() == 'theano': + else: # features before the convolution axis, for weight_len the input and output have to be subtracted axis_filter = -1 - (weight_len - 2) - else: - raise ValueError('Unknown backend: {}'.format(K.backend())) total_featuremaps = shape_act[axis_filter] diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index 89c6bbe..552dbd0 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -96,12 +96,10 @@ def test_DeadDeadReluDetector_conv(): n_samples = 9 # (5, 5) kernel, 4 input featuremaps and 10 output featuremaps - if K.backend() == 'tensorflow': + if K.image_data_format() == 'channels_last': input_shape = (n_samples, 5, 5, 4) - elif K.backend() == 'theano': - input_shape = (n_samples, 4, 5, 5) else: - raise ValueError('Unknown backend: {}'.format(K.backend())) + input_shape = (n_samples, 4, 5, 5) # ignore batch size input_shape_conv = tuple(input_shape[1:]) From 3f206044c8e2f0d21451e777a1badb760dc10e58 Mon Sep 17 00:00:00 2001 From: lameeus Date: Fri, 27 Oct 2017 11:25:07 +0200 Subject: [PATCH 06/29] changed warning method to be adjustable assert replaced by raise Error --- keras_contrib/callbacks/dead_relu_detector.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/keras_contrib/callbacks/dead_relu_detector.py b/keras_contrib/callbacks/dead_relu_detector.py index f545ee2..16ea7cc 100644 --- a/keras_contrib/callbacks/dead_relu_detector.py +++ b/keras_contrib/callbacks/dead_relu_detector.py @@ -15,13 +15,17 @@ class DeadReluDetector(Callback): True means that even a single dead neuron triggers warning False means that only significant number of dead neurons (10% or more) triggers warning + bool_warning: output mode + True means a warning is raised + False means the warning message is printed. """ - def __init__(self, x_train, verbose=False): + def __init__(self, x_train, verbose=False, bool_warning = False): super(DeadReluDetector, self).__init__() self.x_train = x_train self.verbose = verbose self.dead_neurons_share_threshold = 0.1 + self.bool_warning = bool_warning @staticmethod def is_relu_layer(layer): @@ -49,7 +53,9 @@ class DeadReluDetector(Callback): # layer_weight is a list [W] (+ [b]) layer_weight = self.model.layers[layer_index].get_weights() # with kernel and bias, the weights are saved as a list [W, b]. If only weights, it is [W] - assert type(layer_weight) == list + if type(layer_weight) is not list: + raise ValueError("'Layer_weight' should be a list, but was {}".format(type(layer_weight))) + layer_weight_shape = np.shape(layer_weight[0]) yield [layer_index, layer_activations, layer_name, layer_weight_shape] @@ -81,5 +87,9 @@ class DeadReluDetector(Callback): if (self.verbose and dead_neurons > 0) or dead_neurons_share > self.dead_neurons_share_threshold: str_warning = 'Layer {} (#{}) has {} dead neurons ({:.2%})!'.format(layer_name, layer_index, dead_neurons, dead_neurons_share) - print(str_warning) - warnings.warn(str_warning, RuntimeWarning) + + if self.bool_warning: + warnings.warn(str_warning, RuntimeWarning) + else: + print(str_warning) + From bbb08f42cbadb7e46962f5de0818bf1d420f30e3 Mon Sep 17 00:00:00 2001 From: lameeus Date: Fri, 27 Oct 2017 11:45:43 +0200 Subject: [PATCH 07/29] Test and PEP8 fixes --- keras_contrib/callbacks/dead_relu_detector.py | 5 ++--- tests/keras_contrib/callbacks/dead_relu_detector_test.py | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/keras_contrib/callbacks/dead_relu_detector.py b/keras_contrib/callbacks/dead_relu_detector.py index 16ea7cc..fedb43f 100644 --- a/keras_contrib/callbacks/dead_relu_detector.py +++ b/keras_contrib/callbacks/dead_relu_detector.py @@ -20,7 +20,7 @@ class DeadReluDetector(Callback): False means the warning message is printed. """ - def __init__(self, x_train, verbose=False, bool_warning = False): + def __init__(self, x_train, verbose=False, bool_warning=False): super(DeadReluDetector, self).__init__() self.x_train = x_train self.verbose = verbose @@ -87,9 +87,8 @@ class DeadReluDetector(Callback): if (self.verbose and dead_neurons > 0) or dead_neurons_share > self.dead_neurons_share_threshold: str_warning = 'Layer {} (#{}) has {} dead neurons ({:.2%})!'.format(layer_name, layer_index, dead_neurons, dead_neurons_share) - + if self.bool_warning: warnings.warn(str_warning, RuntimeWarning) else: print(str_warning) - diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index 552dbd0..edb5383 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -29,7 +29,7 @@ def test_DeadDeadReluDetector(): dataset, np.ones(shape_out), epochs=1, - callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], + callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose, bool_warning=True)], verbose=False ) assert len(w) == expected_warnings @@ -71,7 +71,7 @@ def test_DeadDeadReluDetector_bias(): dataset, np.ones(shape_out), epochs=1, - callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], + callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose, bool_warning=True)], verbose=False ) assert len(w) == expected_warnings @@ -119,7 +119,7 @@ def test_DeadDeadReluDetector_conv(): dataset, np.ones(shape_out), epochs=1, - callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], + callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose, bool_warning=True)], verbose=False ) assert len(w) == expected_warnings From e873325d0b59a8f7c983613af0f00f7bc99284ae Mon Sep 17 00:00:00 2001 From: lameeus Date: Mon, 6 Nov 2017 17:50:59 +0100 Subject: [PATCH 08/29] 1) Extended layers to --- keras_contrib/callbacks/dead_relu_detector.py | 18 +-- .../callbacks/dead_relu_detector_test.py | 126 +++++++++++------- 2 files changed, 82 insertions(+), 62 deletions(-) diff --git a/keras_contrib/callbacks/dead_relu_detector.py b/keras_contrib/callbacks/dead_relu_detector.py index fedb43f..f8f6839 100644 --- a/keras_contrib/callbacks/dead_relu_detector.py +++ b/keras_contrib/callbacks/dead_relu_detector.py @@ -1,5 +1,4 @@ import numpy as np -import warnings from keras.callbacks import Callback from keras import backend as K @@ -12,20 +11,16 @@ class DeadReluDetector(Callback): # Arguments x_train: Training dataset to check whether or not neurons fire verbose: verbosity mode - True means that even a single dead neuron triggers warning + True means that even a single dead neuron triggers a warning message False means that only significant number of dead neurons (10% or more) - triggers warning - bool_warning: output mode - True means a warning is raised - False means the warning message is printed. + triggers a warning message """ - def __init__(self, x_train, verbose=False, bool_warning=False): + def __init__(self, x_train, verbose=False): super(DeadReluDetector, self).__init__() self.x_train = x_train self.verbose = verbose self.dead_neurons_share_threshold = 0.1 - self.bool_warning = bool_warning @staticmethod def is_relu_layer(layer): @@ -84,11 +79,8 @@ class DeadReluDetector(Callback): dead_neurons = np.sum(np.sum(activation_values, axis=axis) == 0) dead_neurons_share = dead_neurons / total_featuremaps - if (self.verbose and dead_neurons > 0) or dead_neurons_share > self.dead_neurons_share_threshold: + if (self.verbose and dead_neurons > 0) or dead_neurons_share >= self.dead_neurons_share_threshold: str_warning = 'Layer {} (#{}) has {} dead neurons ({:.2%})!'.format(layer_name, layer_index, dead_neurons, dead_neurons_share) - if self.bool_warning: - warnings.warn(str_warning, RuntimeWarning) - else: - print(str_warning) + print(str_warning) diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index edb5383..09e6403 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -1,101 +1,124 @@ import pytest -import warnings import numpy as np +import sys +import io from keras_contrib import callbacks from keras.models import Sequential from keras.layers import Dense, Conv2D, Flatten from keras import backend as K +n_out = 11 # with 1 neuron dead, 1/11 is just below the threshold of 10% with verbose = False + + +def check_print(do_train, expected_warnings, nr_dead: int = None, perc_dead: float = None): + """ + :param perc_dead: as float, 10% should be written as 0.1 + Receive stdout to check if correct warning message is delivered. + """ + saved_stdout = sys.stdout + out = io.StringIO() + sys.stdout = out # overwrite current stdout + + do_train() + + stdoutput = out.getvalue() # get prints, can be something like: "Layer dense (#0) has 2 dead neurons (20.00%)!" + sys.stdout = saved_stdout # restore stdout + + str_count = "dead neurons" + count = stdoutput.count(str_count) + assert expected_warnings == count + if expected_warnings and (nr_dead is not None): + assert 'has {} dead'.format(nr_dead) in stdoutput + if expected_warnings and (perc_dead is not None): + assert 'neurons ({:.2%})'.format(perc_dead) in stdoutput + def test_DeadDeadReluDetector(): n_samples = 9 input_shape = (n_samples, 3, 4) # 4 input features - shape_out = (n_samples, 3, 10) # 10 output features - shape_weights = (4, 10) + shape_out = (n_samples, 3, n_out) # 11 output features + shape_weights = (4, n_out) # ignore batch size input_shape_dense = tuple(input_shape[1:]) - def do_test(weights, expected_warnings, verbose): - with warnings.catch_warnings(record=True) as w: + def do_test(weights, expected_warnings, verbose, nr_dead=None, perc_dead=None): + + def do_train(): dataset = np.ones(input_shape) # data to be fed as training model = Sequential() - model.add(Dense(10, activation='relu', input_shape=input_shape_dense, + model.add(Dense(n_out, activation='relu', input_shape=input_shape_dense, use_bias=False, weights=[weights], name='dense')) model.compile(optimizer='sgd', loss='categorical_crossentropy') model.fit( dataset, np.ones(shape_out), epochs=1, - callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose, bool_warning=True)], + callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], verbose=False ) - assert len(w) == expected_warnings - for warn_item in w: - assert issubclass(warn_item.category, RuntimeWarning) - assert "dead neurons" in str(warn_item.message) - weights_1_dead = np.ones(shape_weights) # weights that correspond to NN with 1/10 neurons dead - weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/10 neurons dead + check_print(do_train, expected_warnings, nr_dead, perc_dead) + + weights_1_dead = np.ones(shape_weights) # weights that correspond to NN with 1/11 neurons dead + weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/11 neurons dead weights_1_dead[:, 0] = 0 weights_2_dead[:, 0:2] = 0 - do_test(weights_1_dead, verbose=True, expected_warnings=1) + do_test(weights_1_dead, verbose=True, expected_warnings=1, nr_dead=1, perc_dead=1. / n_out) do_test(weights_1_dead, verbose=False, expected_warnings=0) - do_test(weights_2_dead, verbose=True, expected_warnings=1) + do_test(weights_2_dead, verbose=True, expected_warnings=1, nr_dead=2, perc_dead=2. / n_out) def test_DeadDeadReluDetector_bias(): n_samples = 9 input_shape = (n_samples, 4) # 4 input features - shape_weights = (4, 10) - shape_out = (n_samples, 10) # 10 output features - shape_bias = (10, ) + shape_weights = (4, n_out) + shape_bias = (n_out, ) + shape_out = (n_samples, n_out) # 11 output features # ignore batch size input_shape_dense = tuple(input_shape[1:]) - def do_test(weights, bias, expected_warnings, verbose): - with warnings.catch_warnings(record=True) as w: + def do_test(weights, bias, expected_warnings, verbose, nr_dead=None, perc_dead=None): + + def do_train(): dataset = np.ones(input_shape) # data to be fed as training model = Sequential() - model.add(Dense(10, activation='relu', input_shape=input_shape_dense, + model.add(Dense(n_out, activation='relu', input_shape=input_shape_dense, use_bias=True, weights=[weights, bias], name='dense')) model.compile(optimizer='sgd', loss='categorical_crossentropy') - # model.compile(optimizer=None, loss='categorical_crossentropy') model.fit( dataset, np.ones(shape_out), epochs=1, - callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose, bool_warning=True)], + callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], verbose=False ) - assert len(w) == expected_warnings - for warn_item in w: - assert issubclass(warn_item.category, RuntimeWarning) - assert "dead neurons" in str(warn_item.message) - weights_1_dead = np.ones(shape_weights) # weights that correspond to NN with 1/10 neurons dead - weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/10 neurons dead + check_print(do_train, expected_warnings, nr_dead, perc_dead) + + weights_1_dead = np.ones(shape_weights) # weights that correspond to NN with 1/11 neurons dead + weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/11 neurons dead weights_1_dead[:, 0] = 0 weights_2_dead[:, 0:2] = 0 bias = np.zeros(shape_bias) - do_test(weights_1_dead, bias, verbose=True, expected_warnings=1) + do_test(weights_1_dead, bias, verbose=True, expected_warnings=1, nr_dead=1, perc_dead=1. / n_out) do_test(weights_1_dead, bias, verbose=False, expected_warnings=0) - do_test(weights_2_dead, bias, verbose=True, expected_warnings=1) + do_test(weights_2_dead, bias, verbose=True, expected_warnings=1, nr_dead=2, perc_dead=2. / n_out) def test_DeadDeadReluDetector_conv(): n_samples = 9 - # (5, 5) kernel, 4 input featuremaps and 10 output featuremaps + # (5, 5) kernel, 4 input featuremaps and 11 output featuremaps if K.image_data_format() == 'channels_last': input_shape = (n_samples, 5, 5, 4) else: @@ -103,43 +126,48 @@ def test_DeadDeadReluDetector_conv(): # ignore batch size input_shape_conv = tuple(input_shape[1:]) - shape_weights = (5, 5, 4, 10) - shape_out = (n_samples, 10) + shape_weights = (5, 5, 4, n_out) + shape_out = (n_samples, n_out) - def do_test(weights_bias, expected_warnings, verbose): - with warnings.catch_warnings(record=True) as w: + def do_test(weights_bias, expected_warnings, verbose, nr_dead: int = None, perc_dead: float = None): + """ + :param perc_dead: as float, 10% should be written as 0.1 + """ - dataset = np.ones(input_shape) # data to be fed as training + def do_train(): + dataset = np.ones(input_shape) # data to be fed as training model = Sequential() - model.add(Conv2D(10, (5, 5), activation='relu', input_shape=input_shape_conv, + model.add(Conv2D(n_out, (5, 5), activation='relu', input_shape=input_shape_conv, use_bias=True, weights=weights_bias, name='conv')) - model.add(Flatten()) # to handle Theano's categorical crossentropy + model.add(Flatten()) # to handle Theano's categorical crossentropy model.compile(optimizer='sgd', loss='categorical_crossentropy') model.fit( dataset, np.ones(shape_out), epochs=1, - callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose, bool_warning=True)], + callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], verbose=False ) - assert len(w) == expected_warnings - for warn_item in w: - assert issubclass(warn_item.category, RuntimeWarning) - assert "dead neurons" in str(warn_item.message) - weights_1_dead = np.ones(shape_weights) # weights that correspond to NN with 1/10 neurons dead + check_print(do_train, expected_warnings, nr_dead, perc_dead) + + weights_1_dead = np.ones(shape_weights) # weights that correspond to NN with 1/11 neurons dead weights_1_dead[..., 0] = 0 - weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/10 neurons dead + weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/11 neurons dead weights_2_dead[..., 0:2] = 0 + weights_all_dead = np.ones(shape_weights) # weights that correspond to NN with all neurons dead + weights_all_dead[..., :] = 0 - bias = np.zeros((10, )) + bias = np.zeros((11, )) weights_bias_1_dead = [weights_1_dead, bias] weights_bias_2_dead = [weights_2_dead, bias] + weights_bias_all_dead = [weights_all_dead, bias] - do_test(weights_bias_1_dead, verbose=True, expected_warnings=1) + do_test(weights_bias_1_dead, verbose=True, expected_warnings=1, nr_dead=1, perc_dead=1. / n_out) do_test(weights_bias_1_dead, verbose=False, expected_warnings=0) - do_test(weights_bias_2_dead, verbose=True, expected_warnings=1) + do_test(weights_bias_2_dead, verbose=True, expected_warnings=1, nr_dead=2, perc_dead=2. / n_out) + do_test(weights_bias_all_dead, verbose=True, expected_warnings=1, nr_dead=11, perc_dead=1.) if __name__ == '__main__': From 9238838aaeb7ce08b5f6c967fc49eec0f8fdcb55 Mon Sep 17 00:00:00 2001 From: lameeus Date: Mon, 6 Nov 2017 17:52:47 +0100 Subject: [PATCH 09/29] As discussed. The 'warning' is now just a print. Test is adjusted to handle printing instead of throwing a warning. --- tests/keras_contrib/callbacks/dead_relu_detector_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index 09e6403..537f0a3 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -13,8 +13,8 @@ n_out = 11 # with 1 neuron dead, 1/11 is just below the threshold of 10% with v def check_print(do_train, expected_warnings, nr_dead: int = None, perc_dead: float = None): """ + Receive stdout to check if correct warning message is delivered :param perc_dead: as float, 10% should be written as 0.1 - Receive stdout to check if correct warning message is delivered. """ saved_stdout = sys.stdout out = io.StringIO() From 148a4f9de650495111e961bf4958bf9e439ecfa1 Mon Sep 17 00:00:00 2001 From: lameeus Date: Mon, 6 Nov 2017 18:13:46 +0100 Subject: [PATCH 10/29] Making test compatible with python 2 and hoping an unexpected bug is fixed (didn't get it at my system) --- .../callbacks/dead_relu_detector_test.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index 537f0a3..c992a8a 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -11,10 +11,11 @@ from keras import backend as K n_out = 11 # with 1 neuron dead, 1/11 is just below the threshold of 10% with verbose = False -def check_print(do_train, expected_warnings, nr_dead: int = None, perc_dead: float = None): +def check_print(do_train, expected_warnings, nr_dead=None, perc_dead=None): """ Receive stdout to check if correct warning message is delivered - :param perc_dead: as float, 10% should be written as 0.1 + :param nr_dead: int + :param perc_dead: float, 10% should be written as 0.1 """ saved_stdout = sys.stdout out = io.StringIO() @@ -23,16 +24,17 @@ def check_print(do_train, expected_warnings, nr_dead: int = None, perc_dead: flo do_train() stdoutput = out.getvalue() # get prints, can be something like: "Layer dense (#0) has 2 dead neurons (20.00%)!" + str_to_count = "dead neurons" + count = stdoutput.count(str_to_count) + sys.stdout = saved_stdout # restore stdout - - str_count = "dead neurons" - count = stdoutput.count(str_count) + assert expected_warnings == count if expected_warnings and (nr_dead is not None): assert 'has {} dead'.format(nr_dead) in stdoutput if expected_warnings and (perc_dead is not None): assert 'neurons ({:.2%})'.format(perc_dead) in stdoutput - + def test_DeadDeadReluDetector(): n_samples = 9 @@ -167,7 +169,7 @@ def test_DeadDeadReluDetector_conv(): do_test(weights_bias_1_dead, verbose=True, expected_warnings=1, nr_dead=1, perc_dead=1. / n_out) do_test(weights_bias_1_dead, verbose=False, expected_warnings=0) do_test(weights_bias_2_dead, verbose=True, expected_warnings=1, nr_dead=2, perc_dead=2. / n_out) - do_test(weights_bias_all_dead, verbose=True, expected_warnings=1, nr_dead=11, perc_dead=1.) + do_test(weights_bias_all_dead, verbose=True, expected_warnings=1, nr_dead=n_out, perc_dead=1.) if __name__ == '__main__': From 46f7a4604914546fedcb8f9bb4cc72165b7760c9 Mon Sep 17 00:00:00 2001 From: lameeus Date: Mon, 6 Nov 2017 18:23:13 +0100 Subject: [PATCH 11/29] Made compatible with python 2.7 and tried to fix weird bug I didn't encounter at own system --- .../callbacks/dead_relu_detector_test.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index c992a8a..9cc38af 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -26,15 +26,17 @@ def check_print(do_train, expected_warnings, nr_dead=None, perc_dead=None): stdoutput = out.getvalue() # get prints, can be something like: "Layer dense (#0) has 2 dead neurons (20.00%)!" str_to_count = "dead neurons" count = stdoutput.count(str_to_count) - + sys.stdout = saved_stdout # restore stdout - + assert expected_warnings == count if expected_warnings and (nr_dead is not None): - assert 'has {} dead'.format(nr_dead) in stdoutput + str_to_check = 'has {} dead'.format(nr_dead) + assert str_to_check in stdoutput, '"{}" not in "{}"!'.format(str_to_check, stdoutput) if expected_warnings and (perc_dead is not None): - assert 'neurons ({:.2%})'.format(perc_dead) in stdoutput - + str_to_check = 'neurons ({:.2%})'.format(perc_dead) + assert str_to_check in stdoutput, '"{}" not in "{}"!'.format(str_to_check, stdoutput) + def test_DeadDeadReluDetector(): n_samples = 9 @@ -173,4 +175,5 @@ def test_DeadDeadReluDetector_conv(): if __name__ == '__main__': - pytest.main([__file__]) + # pytest.main([__file__]) + test_DeadDeadReluDetector_conv() From b2520b935ddef481499966c6ac4ecde28264601d Mon Sep 17 00:00:00 2001 From: lameeus Date: Mon, 6 Nov 2017 18:23:43 +0100 Subject: [PATCH 12/29] forgot to remove comment --- tests/keras_contrib/callbacks/dead_relu_detector_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index 9cc38af..134be70 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -175,5 +175,4 @@ def test_DeadDeadReluDetector_conv(): if __name__ == '__main__': - # pytest.main([__file__]) test_DeadDeadReluDetector_conv() From f975a52ea91334d0430d6e1be0a03c2e80a8a408 Mon Sep 17 00:00:00 2001 From: lameeus Date: Mon, 6 Nov 2017 18:47:22 +0100 Subject: [PATCH 13/29] Second try. For some reason it doesn't print with all weights 0 at server side. --- tests/keras_contrib/callbacks/dead_relu_detector_test.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index 134be70..84f1b91 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -133,7 +133,7 @@ def test_DeadDeadReluDetector_conv(): shape_weights = (5, 5, 4, n_out) shape_out = (n_samples, n_out) - def do_test(weights_bias, expected_warnings, verbose, nr_dead: int = None, perc_dead: float = None): + def do_test(weights_bias, expected_warnings, verbose, nr_dead = None, perc_dead = None): """ :param perc_dead: as float, 10% should be written as 0.1 """ @@ -159,8 +159,7 @@ def test_DeadDeadReluDetector_conv(): weights_1_dead[..., 0] = 0 weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/11 neurons dead weights_2_dead[..., 0:2] = 0 - weights_all_dead = np.ones(shape_weights) # weights that correspond to NN with all neurons dead - weights_all_dead[..., :] = 0 + weights_all_dead = np.zeros(shape_weights) # weights that correspond to NN with all neurons dead bias = np.zeros((11, )) @@ -171,8 +170,8 @@ def test_DeadDeadReluDetector_conv(): do_test(weights_bias_1_dead, verbose=True, expected_warnings=1, nr_dead=1, perc_dead=1. / n_out) do_test(weights_bias_1_dead, verbose=False, expected_warnings=0) do_test(weights_bias_2_dead, verbose=True, expected_warnings=1, nr_dead=2, perc_dead=2. / n_out) - do_test(weights_bias_all_dead, verbose=True, expected_warnings=1, nr_dead=n_out, perc_dead=1.) + do_test(weights_bias_all_dead, verbose=True, expected_warnings=1, nr_dead=n_out, perc_dead=n_out / n_out) if __name__ == '__main__': - test_DeadDeadReluDetector_conv() + pytest.main([__file__]) From 9af7625e0f704766950c925eed14379e47748f76 Mon Sep 17 00:00:00 2001 From: lameeus Date: Mon, 6 Nov 2017 19:26:50 +0100 Subject: [PATCH 14/29] fixed python 2 bug with divison StringIO depends now on python 2/3 as it should PEP8 fixes --- keras_contrib/callbacks/dead_relu_detector.py | 2 +- .../callbacks/dead_relu_detector_test.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/keras_contrib/callbacks/dead_relu_detector.py b/keras_contrib/callbacks/dead_relu_detector.py index f8f6839..2cfe37b 100644 --- a/keras_contrib/callbacks/dead_relu_detector.py +++ b/keras_contrib/callbacks/dead_relu_detector.py @@ -78,7 +78,7 @@ class DeadReluDetector(Callback): dead_neurons = np.sum(np.sum(activation_values, axis=axis) == 0) - dead_neurons_share = dead_neurons / total_featuremaps + dead_neurons_share = float(dead_neurons) / float(total_featuremaps) if (self.verbose and dead_neurons > 0) or dead_neurons_share >= self.dead_neurons_share_threshold: str_warning = 'Layer {} (#{}) has {} dead neurons ({:.2%})!'.format(layer_name, layer_index, dead_neurons, dead_neurons_share) diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index 84f1b91..e4a1783 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -1,7 +1,11 @@ import pytest import numpy as np import sys -import io + +if (sys.version_info > (3, 0)): + from io import StringIO +else: + from StringIO import StringIO from keras_contrib import callbacks from keras.models import Sequential @@ -17,8 +21,10 @@ def check_print(do_train, expected_warnings, nr_dead=None, perc_dead=None): :param nr_dead: int :param perc_dead: float, 10% should be written as 0.1 """ + saved_stdout = sys.stdout - out = io.StringIO() + + out = StringIO() sys.stdout = out # overwrite current stdout do_train() @@ -133,7 +139,7 @@ def test_DeadDeadReluDetector_conv(): shape_weights = (5, 5, 4, n_out) shape_out = (n_samples, n_out) - def do_test(weights_bias, expected_warnings, verbose, nr_dead = None, perc_dead = None): + def do_test(weights_bias, expected_warnings, verbose, nr_dead=None, perc_dead=None): """ :param perc_dead: as float, 10% should be written as 0.1 """ From 58d1817d4cf6797ceccc9528f663c714132985a5 Mon Sep 17 00:00:00 2001 From: lameeus Date: Mon, 6 Nov 2017 19:29:00 +0100 Subject: [PATCH 15/29] PEP8 --- tests/keras_contrib/callbacks/dead_relu_detector_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index e4a1783..4026890 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -21,7 +21,7 @@ def check_print(do_train, expected_warnings, nr_dead=None, perc_dead=None): :param nr_dead: int :param perc_dead: float, 10% should be written as 0.1 """ - + saved_stdout = sys.stdout out = StringIO() From a1079c012a69171cef501c39704303adfdafb8a9 Mon Sep 17 00:00:00 2001 From: lameeus Date: Tue, 7 Nov 2017 11:10:39 +0100 Subject: [PATCH 16/29] I still can't reproduce the bug I get at server side. I commented those lines of code. If someone else could find out what is going wrong? --- .../callbacks/dead_relu_detector_test.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/keras_contrib/callbacks/dead_relu_detector_test.py b/tests/keras_contrib/callbacks/dead_relu_detector_test.py index 4026890..5f7c396 100644 --- a/tests/keras_contrib/callbacks/dead_relu_detector_test.py +++ b/tests/keras_contrib/callbacks/dead_relu_detector_test.py @@ -25,23 +25,25 @@ def check_print(do_train, expected_warnings, nr_dead=None, perc_dead=None): saved_stdout = sys.stdout out = StringIO() + out.flush() sys.stdout = out # overwrite current stdout do_train() - stdoutput = out.getvalue() # get prints, can be something like: "Layer dense (#0) has 2 dead neurons (20.00%)!" + stdoutput = out.getvalue().strip() # get prints, can be something like: "Layer dense (#0) has 2 dead neurons (20.00%)!" str_to_count = "dead neurons" count = stdoutput.count(str_to_count) sys.stdout = saved_stdout # restore stdout + out.close() assert expected_warnings == count if expected_warnings and (nr_dead is not None): str_to_check = 'has {} dead'.format(nr_dead) - assert str_to_check in stdoutput, '"{}" not in "{}"!'.format(str_to_check, stdoutput) + assert str_to_check in stdoutput, '"{}" not in "{}"'.format(str_to_check, stdoutput) if expected_warnings and (perc_dead is not None): - str_to_check = 'neurons ({:.2%})'.format(perc_dead) - assert str_to_check in stdoutput, '"{}" not in "{}"!'.format(str_to_check, stdoutput) + str_to_check = 'neurons ({:.2%})!'.format(perc_dead) + assert str_to_check in stdoutput, '"{}" not in "{}"'.format(str_to_check, stdoutput) def test_DeadDeadReluDetector(): @@ -65,6 +67,7 @@ def test_DeadDeadReluDetector(): model.fit( dataset, np.ones(shape_out), + batch_size=1, epochs=1, callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], verbose=False @@ -74,6 +77,7 @@ def test_DeadDeadReluDetector(): weights_1_dead = np.ones(shape_weights) # weights that correspond to NN with 1/11 neurons dead weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/11 neurons dead + weights_all_dead = np.zeros(shape_weights) # weights that correspond to all neurons dead weights_1_dead[:, 0] = 0 weights_2_dead[:, 0:2] = 0 @@ -81,6 +85,7 @@ def test_DeadDeadReluDetector(): do_test(weights_1_dead, verbose=True, expected_warnings=1, nr_dead=1, perc_dead=1. / n_out) do_test(weights_1_dead, verbose=False, expected_warnings=0) do_test(weights_2_dead, verbose=True, expected_warnings=1, nr_dead=2, perc_dead=2. / n_out) + # do_test(weights_all_dead, verbose=True, expected_warnings=1, nr_dead=n_out, perc_dead=1.) def test_DeadDeadReluDetector_bias(): @@ -105,6 +110,7 @@ def test_DeadDeadReluDetector_bias(): model.fit( dataset, np.ones(shape_out), + batch_size=1, epochs=1, callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], verbose=False @@ -114,6 +120,7 @@ def test_DeadDeadReluDetector_bias(): weights_1_dead = np.ones(shape_weights) # weights that correspond to NN with 1/11 neurons dead weights_2_dead = np.ones(shape_weights) # weights that correspond to NN with 2/11 neurons dead + weights_all_dead = np.zeros(shape_weights) # weights that correspond to all neurons dead weights_1_dead[:, 0] = 0 weights_2_dead[:, 0:2] = 0 @@ -123,6 +130,7 @@ def test_DeadDeadReluDetector_bias(): do_test(weights_1_dead, bias, verbose=True, expected_warnings=1, nr_dead=1, perc_dead=1. / n_out) do_test(weights_1_dead, bias, verbose=False, expected_warnings=0) do_test(weights_2_dead, bias, verbose=True, expected_warnings=1, nr_dead=2, perc_dead=2. / n_out) + # do_test(weights_all_dead, bias, verbose=True, expected_warnings=1, nr_dead=n_out, perc_dead=1.) def test_DeadDeadReluDetector_conv(): @@ -154,6 +162,7 @@ def test_DeadDeadReluDetector_conv(): model.fit( dataset, np.ones(shape_out), + batch_size=1, epochs=1, callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)], verbose=False @@ -176,7 +185,7 @@ def test_DeadDeadReluDetector_conv(): do_test(weights_bias_1_dead, verbose=True, expected_warnings=1, nr_dead=1, perc_dead=1. / n_out) do_test(weights_bias_1_dead, verbose=False, expected_warnings=0) do_test(weights_bias_2_dead, verbose=True, expected_warnings=1, nr_dead=2, perc_dead=2. / n_out) - do_test(weights_bias_all_dead, verbose=True, expected_warnings=1, nr_dead=n_out, perc_dead=n_out / n_out) + # do_test(weights_bias_all_dead, verbose=True, expected_warnings=1, nr_dead=n_out, perc_dead=1.) if __name__ == '__main__': From 2cb41dbc039018aeee246a01d002770dbc22b6c7 Mon Sep 17 00:00:00 2001 From: Somshubra Majumdar Date: Thu, 16 Nov 2017 22:27:47 -0600 Subject: [PATCH 17/29] Add NASNet models --- keras_contrib/applications/__init__.py | 3 + keras_contrib/applications/nasnet.py | 558 +++++++++++++++++++++++++ 2 files changed, 561 insertions(+) create mode 100644 keras_contrib/applications/nasnet.py diff --git a/keras_contrib/applications/__init__.py b/keras_contrib/applications/__init__.py index e9d829d..a1592a7 100644 --- a/keras_contrib/applications/__init__.py +++ b/keras_contrib/applications/__init__.py @@ -1,2 +1,5 @@ from .densenet import DenseNet from .ror import ResidualOfResidual +from .resnet import ResNet, ResNet18, ResNet34, ResNet50, ResNet101, ResNet152 +from .wide_resnet import WideResidualNetwork +from .nasnet import NASNet, NASNetLarge, NASNetMobile diff --git a/keras_contrib/applications/nasnet.py b/keras_contrib/applications/nasnet.py new file mode 100644 index 0000000..4ab5210 --- /dev/null +++ b/keras_contrib/applications/nasnet.py @@ -0,0 +1,558 @@ +"""NASNet (Neural Architecture Search Networks) for Keras + +# References + + - [Learning Transferable Architectures for Scalable Image Recognition] + (https://arxiv.org/abs/1707.07012) + +Reference material for extended functionality: + + - https://github.com/tensorflow/models/blob/master/research/slim/nets/ + nasnet/nasnet.py + - https://github.com/taehoonlee/tensornets/blob/master/tensornets/nasnets.py +""" +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division + +import warnings + +from keras.models import Model +from keras.layers import Input +from keras.layers import Activation +from keras.layers import Dense +from keras.layers import Dropout +from keras.layers import BatchNormalization +from keras.layers import MaxPooling2D +from keras.layers import AveragePooling2D +from keras.layers import GlobalAveragePooling2D +from keras.layers import GlobalMaxPooling2D +from keras.layers import Conv2D +from keras.layers import SeparableConv2D +from keras.layers import concatenate +from keras.layers import add +from keras.utils.data_utils import get_file +from keras.engine.topology import get_source_inputs +from keras.applications.imagenet_utils import _obtain_input_shape +from keras.applications.inception_v3 import preprocess_input +from keras.applications.imagenet_utils import decode_predictions +from keras import backend as K + + +_BN_DECAY = 0.9997 +_BN_EPSILON = 1e-3 + + +def NASNet(input_shape=None, + penultimate_filters=4032, + nb_blocks=6, + stem_filters=96, + skip_reduction=True, + use_auxilary_branch=False, + filters_multiplier=2, + dropout=0.5, + include_top=True, + weights='imagenet', + input_tensor=None, + pooling=None, + classes=1000, + default_size=None): + """Instantiates a NASNet architecture. + Note that only TensorFlow is supported for now, + therefore it only works with the data format + `image_data_format='channels_last'` in your Keras config + at `~/.keras/keras.json`. + + # Arguments + input_shape: optional shape tuple, only to be specified + if `include_top` is False (otherwise the input shape + has to be `(331, 331, 3)` for NASNetLarge or + `(224, 224, 3)` for NASNetMobile + It should have exactly 3 inputs channels, + and width and height should be no smaller than 32. + E.g. `(224, 224, 3)` would be one valid value. + penultimate_filters: number of filters in the penultimate layer. + NASNet models use the notation `NASNet (N @ P)`, where: + - N is the number of blocks + - P is the number of penultimate filters + nb_blocks: number of repeated blocks of the NASNet model. + NASNet models use the notation `NASNet (N @ P)`, where: + - N is the number of blocks + - P is the number of penultimate filters + stem_filters: number of filters in the initial stem block + skip_reduction: Whether to skip the reduction step at the tail + end of the network. Set to `False` for CIFAR models. + use_auxilary_branch: Whether to use the auxilary branch during + training or evaluation. + filters_multiplier: controls the width of the network. + - If `filters_multiplier` < 1.0, proportionally decreases the number + of filters in each layer. + - If `filters_multiplier` > 1.0, proportionally increases the number + of filters in each layer. + - If `filters_multiplier` = 1, default number of filters from the paper + are used at each layer. + dropout: dropout rate + include_top: whether to include the fully-connected + layer at the top of the network. + weights: `None` (random initialization) or + `imagenet` (ImageNet weights) + input_tensor: optional Keras tensor (i.e. output of + `layers.Input()`) + to use as image input for the model. + pooling: Optional pooling mode for feature extraction + when `include_top` is `False`. + - `None` means that the output of the model + will be the 4D tensor output of the + last convolutional layer. + - `avg` means that global average pooling + will be applied to the output of the + last convolutional layer, and thus + the output of the model will be a + 2D tensor. + - `max` means that global max pooling will + be applied. + classes: optional number of classes to classify images + into, only to be specified if `include_top` is True, and + if no `weights` argument is specified. + default_size: specifies the default image size of the model + # Returns + A Keras model instance. + # Raises + ValueError: in case of invalid argument for `weights`, + or invalid input shape. + RuntimeError: If attempting to run this model with a + backend that does not support separable convolutions. + """ + if K.backend() != 'tensorflow': + raise RuntimeError('Only Tensorflow backend is currently supported, ' + 'as other backends do not support ' + 'separable convolution.') + + if weights not in {'imagenet', None}: + raise ValueError('The `weights` argument should be either ' + '`None` (random initialization) or `imagenet` ' + '(pre-training on ImageNet).') + + if weights == 'imagenet' and include_top and classes != 1000: + raise ValueError('If using `weights` as ImageNet with `include_top` ' + 'as true, `classes` should be 1000') + + if default_size is None: + default_size = 331 + + # Determine proper input shape and default size. + input_shape = _obtain_input_shape(input_shape, + default_size=default_size, + min_size=32, + data_format=K.image_data_format(), + require_flatten=include_top or weights) + + if K.image_data_format() != 'channels_last': + warnings.warn('The MobileNet family of models is only available ' + 'for the input data format "channels_last" ' + '(width, height, channels). ' + 'However your settings specify the default ' + 'data format "channels_first" (channels, width, height).' + ' You should set `image_data_format="channels_last"` ' + 'in your Keras config located at ~/.keras/keras.json. ' + 'The model being returned right now will expect inputs ' + 'to follow the "channels_last" data format.') + K.set_image_data_format('channels_last') + old_data_format = 'channels_first' + else: + old_data_format = None + + if input_tensor is None: + img_input = Input(shape=input_shape) + else: + if not K.is_keras_tensor(input_tensor): + img_input = Input(tensor=input_tensor, shape=input_shape) + else: + img_input = input_tensor + + assert penultimate_filters % ((2 ** nb_blocks) * 6), "`penultimate_filters` needs to be divisible " \ + "by 6 * (2^N)." + + filters = penultimate_filters // ((2 ** nb_blocks) * 6) + + x = Conv2D(stem_filters, (3, 3), strides=(2, 2), padding='valid', use_bias=False, name='stem_conv1', + kernel_initializer='he_normal')(img_input) + + x, p = _reduction_A(x, None, filters // (filters_multiplier ** 2), id='stem_1') + x, p = _reduction_A(x, p, filters // filters_multiplier, id='stem_2') + + for i in range(nb_blocks): + x, p = _normal_A(x, p, filters, id='%d' % (i)) + + x, p0 = _reduction_A(x, p, filters * filters_multiplier, id='reduce_%d' % (nb_blocks)) + + p = p0 if not skip_reduction else p + + for i in range(nb_blocks): + x, p = _normal_A(x, p, filters * filters_multiplier, id='%d' % (nb_blocks + i + 1)) + + auxilary_x = None + if use_auxilary_branch: + channel_dim = 1 if K.image_data_format() == 'channels_first' else -1 + img_dim = 2 if K.image_data_format() == 'channels_first' else -2 + + auxilary_x = Activation('relu')(x) + auxilary_x = AveragePooling2D((5, 5), strides=(3, 3), padding='valid', name='aux_pool')(auxilary_x) + auxilary_x = Conv2D(128, (1, 1), padding='same', use_bias=False, name='aux_conv_projection', + kernel_initializer='he_normal')(auxilary_x) + auxilary_x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name='aux_bn_projection')(auxilary_x) + auxilary_x = Activation('relu')(auxilary_x) + + auxilary_x = Conv2D(768, auxilary_x._keras_shape[img_dim], padding='valid', use_bias=False, + kernel_initializer='he_normal', name='aux_conv_reduction')(auxilary_x) + auxilary_x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name='aux_bn_reduction')(auxilary_x) + auxilary_x = Activation('relu')(auxilary_x) + + auxilary_x = GlobalAveragePooling2D()(auxilary_x) + auxilary_x = Dense(classes, activation='softmax')(auxilary_x) + + x, p0 = _reduction_A(x, p, filters * filters_multiplier ** 2, id='reduce_%d' % (2 * nb_blocks)) + + p = p0 if not skip_reduction else p + + for i in range(nb_blocks): + x, p = _normal_A(x, p, filters * filters_multiplier ** 2, id='%d' % (2 * nb_blocks + i + 1)) + + x = Activation('relu')(x) + + if include_top: + x = GlobalAveragePooling2D()(x) + x = Dropout(dropout)(x) + x = Dense(classes, activation='softmax')(x) + else: + if pooling == 'avg': + x = GlobalAveragePooling2D()(x) + elif pooling == 'max': + x = GlobalMaxPooling2D()(x) + + # Ensure that the model takes into account + # any potential predecessors of `input_tensor`. + if input_tensor is not None: + inputs = get_source_inputs(input_tensor) + else: + inputs = img_input + + # Create model. + if use_auxilary_branch: + model = Model(inputs, [x, auxilary_x], name='NASNet_with_auxilary') + else: + model = Model(inputs, x, name='NASNet') + + # load weights (when available) + warnings.warn('Weights of NASNet models have not been ported yet for Keras.') + + if old_data_format: + K.set_image_data_format(old_data_format) + + return model + + +def NASNetLarge(input_shape=None, + dropout=0.5, + use_auxilary_branch=False, + include_top=True, + weights='imagenet', + input_tensor=None, + pooling=None, + classes=1000): + """Instantiates a NASNet architecture in ImageNet mode. + Note that only TensorFlow is supported for now, + therefore it only works with the data format + `image_data_format='channels_last'` in your Keras config + at `~/.keras/keras.json`. + + # Arguments + input_shape: optional shape tuple, only to be specified + if `include_top` is False (otherwise the input shape + has to be `(331, 331, 3)` for NASNetLarge. + It should have exactly 3 inputs channels, + and width and height should be no smaller than 32. + E.g. `(224, 224, 3)` would be one valid value. + use_auxilary_branch: Whether to use the auxilary branch during + training or evaluation. + dropout: dropout rate + include_top: whether to include the fully-connected + layer at the top of the network. + weights: `None` (random initialization) or + `imagenet` (ImageNet weights) + input_tensor: optional Keras tensor (i.e. output of + `layers.Input()`) + to use as image input for the model. + pooling: Optional pooling mode for feature extraction + when `include_top` is `False`. + - `None` means that the output of the model + will be the 4D tensor output of the + last convolutional layer. + - `avg` means that global average pooling + will be applied to the output of the + last convolutional layer, and thus + the output of the model will be a + 2D tensor. + - `max` means that global max pooling will + be applied. + classes: optional number of classes to classify images + into, only to be specified if `include_top` is True, and + if no `weights` argument is specified. + default_size: specifies the default image size of the model + # Returns + A Keras model instance. + # Raises + ValueError: in case of invalid argument for `weights`, + or invalid input shape. + RuntimeError: If attempting to run this model with a + backend that does not support separable convolutions. + """ + return NASNet(input_shape, + penultimate_filters=4032, + nb_blocks=6, + stem_filters=96, + skip_reduction=True, + use_auxilary_branch=use_auxilary_branch, + filters_multiplier=2, + dropout=dropout, + include_top=include_top, + weights=weights, + input_tensor=input_tensor, + pooling=pooling, + classes=classes, + default_size=331) + + +def NASNetMobile(input_shape=None, + dropout=0.5, + use_auxilary_branch=False, + include_top=True, + weights='imagenet', + input_tensor=None, + pooling=None, + classes=1000): + """Instantiates a NASNet architecture in CIFAR mode. + Note that only TensorFlow is supported for now, + therefore it only works with the data format + `image_data_format='channels_last'` in your Keras config + at `~/.keras/keras.json`. + + # Arguments + input_shape: optional shape tuple, only to be specified + if `include_top` is False (otherwise the input shape + has to be `(224, 224, 3)` for NASNetMobile + It should have exactly 3 inputs channels, + and width and height should be no smaller than 32. + E.g. `(224, 224, 3)` would be one valid value. + use_auxilary_branch: Whether to use the auxilary branch during + training or evaluation. + dropout: dropout rate + include_top: whether to include the fully-connected + layer at the top of the network. + weights: `None` (random initialization) or + `imagenet` (ImageNet weights) + input_tensor: optional Keras tensor (i.e. output of + `layers.Input()`) + to use as image input for the model. + pooling: Optional pooling mode for feature extraction + when `include_top` is `False`. + - `None` means that the output of the model + will be the 4D tensor output of the + last convolutional layer. + - `avg` means that global average pooling + will be applied to the output of the + last convolutional layer, and thus + the output of the model will be a + 2D tensor. + - `max` means that global max pooling will + be applied. + classes: optional number of classes to classify images + into, only to be specified if `include_top` is True, and + if no `weights` argument is specified. + default_size: specifies the default image size of the model + # Returns + A Keras model instance. + # Raises + ValueError: in case of invalid argument for `weights`, + or invalid input shape. + RuntimeError: If attempting to run this model with a + backend that does not support separable convolutions. + """ + return NASNet(input_shape, + penultimate_filters=1056, + nb_blocks=4, + stem_filters=32, + skip_reduction=False, + use_auxilary_branch=use_auxilary_branch, + filters_multiplier=2, + dropout=dropout, + include_top=include_top, + weights=weights, + input_tensor=input_tensor, + pooling=pooling, + classes=classes, + default_size=224) + + +def _separable_conv_block(ip, filters, kernel_size=(3, 3), strides=(1, 1), id=None): + '''Adds 2 blocks of [relu-separable conv-batchnorm] + + # Arguments: + ip: input tensor + filters: number of output filters per layer + kernel_size: kernel size of separable convolutions + strides: strided convolution for downsampling + id: string id + + # Returns: + a Keras tensor + ''' + channel_dim = 1 if K.image_data_format() == 'channels_first' else -1 + + x = Activation('relu')(ip) + x = SeparableConv2D(filters, kernel_size, strides=strides, name='separable_conv_1_%s' % id, + padding='same', use_bias=False, kernel_initializer='he_normal')(x) + x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name="separable_conv_1_bn_%s" % (id))(x) + x = Activation('relu')(x) + x = SeparableConv2D(filters, kernel_size, name='separable_conv_2_%s' % id, + padding='same', use_bias=False, kernel_initializer='he_normal')(x) + x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name="separable_conv_2_bn_%s" % (id))(x) + return x + + +def _adjust_block(p, ip, filters, id=None): + ''' + Adjusts the input `p` to match the shape of the `input` + or situations where the output number of filters needs to + be changed + + # Arguments: + p: input tensor which needs to be modified + ip: input tensor whose shape needs to be matched + filters: number of output filters to be matched + id: string id + + # Returns: + an adjusted Keras tensor + ''' + channel_dim = 1 if K.image_data_format() == 'channels_first' else -1 + img_dim = 2 if K.image_data_format() == 'channels_first' else -2 + + if p is None: + p = ip + + elif p._keras_shape[img_dim] != ip._keras_shape[img_dim]: + p = Activation('relu', name='adjust_relu_1_%s' % id)(p) + + p1 = AveragePooling2D((1, 1), strides=(2, 2), padding='valid', name='adjust_avg_pool_1_%s' % id)(p) + p1 = Conv2D(filters // 2, (1, 1), padding='same', use_bias=False, + name='adjust_conv_1_%s' % id, kernel_initializer='he_normal')(p1) + + p2 = AveragePooling2D((1, 1), strides=(2, 2), padding='valid', name='adjust_avg_pool_2_%s' % id)(p) + p2 = Conv2D(filters // 2, (1, 1), padding='same', use_bias=False, + name='adjust_conv_2_%s' % id, kernel_initializer='he_normal')(p2) + + p = concatenate([p1, p2], axis=channel_dim) + p = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name='adjust_bn_%s' % id)(p) + + elif p._keras_shape[channel_dim] != filters: + p = Activation('relu')(p) + p = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='adjust_conv_projection_%s' % id, + use_bias=False, kernel_initializer='he_normal')(p) + p = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name='adjust_bn_%s' % id)(p) + return p + + +def _normal_A(ip, p, filters, id=None): + '''Adds a Normal cell for NASNet-A (Fig. 4 in the paper) + + # Arguments: + ip: input tensor `x` + p: input tensor `p` + filters: number of output filters + id: string id + + # Returns: + a Keras tensor + ''' + channel_dim = 1 if K.image_data_format() == 'channels_first' else -1 + + p = _adjust_block(p, ip, filters, id) + + h = Activation('relu')(ip) + h = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='normal_conv_1_%s' % id, + use_bias=False, kernel_initializer='he_normal')(h) + h = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name='normal_bn_1_%s' % id)(h) + + x1 = _separable_conv_block(h, filters, id='normal_left1_%s' % id) + x1 = add([x1, h], name='normal_add_1_%s' % id) + + x2_1 = _separable_conv_block(p, filters, id='normal_left2_%s' % id) + x2_2 = _separable_conv_block(h, filters, kernel_size=(5, 5), id='normal_right2_%s' % id) + x2 = add([x2_1, x2_2], name='normal_add_2_%s' % id) + + x3 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_left3_%s' % (id))(h) + x3 = add([x3, p], name='normal_add_3_%s' % id) + + x4_1 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_left4_%s' % (id))(p) + x4_2 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_right4_%s' % (id))(p) + x4 = add([x4_1, x4_2], name='normal_add_4_%s' % id) + + x5_1 = _separable_conv_block(p, filters, (5, 5), id='normal_left5_%s' % id) + x5_2 = _separable_conv_block(p, filters, (3, 3), id='normal_right5_%s' % id) + x5 = add([x5_1, x5_2], name='normal_add_5_%s' % id) + + x = concatenate([p, x2, x5, x3, x4, x1], axis=channel_dim, name='normal_concat_%s' % id) + return x, ip + + +def _reduction_A(ip, p, filters, id=None): + '''Adds a Reduction cell for NASNet-A (Fig. 4 in the paper) + + # Arguments: + ip: input tensor `x` + p: input tensor `p` + filters: number of output filters + id: string id + + # Returns: + a Keras tensor + ''' + """""" + channel_dim = 1 if K.image_data_format() == 'channels_first' else -1 + + p = _adjust_block(p, ip, filters, id) + + h = Activation('relu')(ip) + h = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='reduction_conv_1_%s' % id, + use_bias=False, kernel_initializer='he_normal')(h) + h = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name='reduction_bn_1_%s' % id)(h) + + x1_1 = _separable_conv_block(p, filters, (7, 7), strides=(2, 2), id='reduction_left1_%s' % id) + x1_2 = _separable_conv_block(h, filters, (5, 5), strides=(2, 2), id='reduction_right1_%s' % id) + x1 = add([x1_1, x1_2], name='reduction_add_1_%s' % id) + + x2_1 = MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left2_%s' % id)(h) + x2_2 = _separable_conv_block(p, filters, (7, 7), strides=(2, 2), id='reduction_right2_%s' % id) + x2 = add([x2_1, x2_2], name='reduction_add_2_%s' % id) + + x3_1 = AveragePooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left3_%s' % id)(h) + x3_2 = _separable_conv_block(p, filters, (5, 5), strides=(2, 2), id='reduction_right3_%s' % id) + x3 = add([x3_1, x3_2], name='reduction_add3_%s' % id) + + x4_1 = MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left4_%s' % id)(h) + x4_2 = _separable_conv_block(x1, filters, (3, 3), id='reduction_right4_%s' % id) + x4 = add([x4_1, x4_2], name='reduction_add4_%s' % id) + + x5 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='reduction_left5_%s' % id)(x1) + + x = concatenate([x2, x3, x5, x4], axis=channel_dim, name='reduction_concat_%s' % id) + return x, ip From 03e649dbca7b1e6c59e52f250f3ca8ea728e560c Mon Sep 17 00:00:00 2001 From: Somshubra Majumdar Date: Fri, 17 Nov 2017 12:32:22 -0600 Subject: [PATCH 18/29] Add name scopes and correct a flaw with assertion --- keras_contrib/applications/nasnet.py | 134 +++++++++++++++------------ 1 file changed, 74 insertions(+), 60 deletions(-) diff --git a/keras_contrib/applications/nasnet.py b/keras_contrib/applications/nasnet.py index 4ab5210..c7f52f8 100644 --- a/keras_contrib/applications/nasnet.py +++ b/keras_contrib/applications/nasnet.py @@ -1,14 +1,14 @@ -"""NASNet (Neural Architecture Search Networks) for Keras - -# References +"""Collection of NASNet models +The reference paper: - [Learning Transferable Architectures for Scalable Image Recognition] (https://arxiv.org/abs/1707.07012) -Reference material for extended functionality: - +The reference implementation: +1. TF Slim - https://github.com/tensorflow/models/blob/master/research/slim/nets/ nasnet/nasnet.py +2. TensorNets - https://github.com/taehoonlee/tensornets/blob/master/tensornets/nasnets.py """ from __future__ import print_function @@ -170,13 +170,15 @@ def NASNet(input_shape=None, else: img_input = input_tensor - assert penultimate_filters % ((2 ** nb_blocks) * 6), "`penultimate_filters` needs to be divisible " \ - "by 6 * (2^N)." + assert penultimate_filters % 24 == 0, "`penultimate_filters` needs to be divisible by 24" - filters = penultimate_filters // ((2 ** nb_blocks) * 6) + channel_dim = 1 if K.image_data_format() == 'channels_first' else -1 + filters = penultimate_filters // 24 x = Conv2D(stem_filters, (3, 3), strides=(2, 2), padding='valid', use_bias=False, name='stem_conv1', kernel_initializer='he_normal')(img_input) + x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name='stem_bn1')(x) x, p = _reduction_A(x, None, filters // (filters_multiplier ** 2), id='stem_1') x, p = _reduction_A(x, p, filters // filters_multiplier, id='stem_2') @@ -193,7 +195,6 @@ def NASNet(input_shape=None, auxilary_x = None if use_auxilary_branch: - channel_dim = 1 if K.image_data_format() == 'channels_first' else -1 img_dim = 2 if K.image_data_format() == 'channels_first' else -2 auxilary_x = Activation('relu')(x) @@ -411,16 +412,17 @@ def _separable_conv_block(ip, filters, kernel_size=(3, 3), strides=(1, 1), id=No ''' channel_dim = 1 if K.image_data_format() == 'channels_first' else -1 - x = Activation('relu')(ip) - x = SeparableConv2D(filters, kernel_size, strides=strides, name='separable_conv_1_%s' % id, - padding='same', use_bias=False, kernel_initializer='he_normal')(x) - x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, - name="separable_conv_1_bn_%s" % (id))(x) - x = Activation('relu')(x) - x = SeparableConv2D(filters, kernel_size, name='separable_conv_2_%s' % id, - padding='same', use_bias=False, kernel_initializer='he_normal')(x) - x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, - name="separable_conv_2_bn_%s" % (id))(x) + with K.name_scope('separable_conv_block_%s' % id): + x = Activation('relu')(ip) + x = SeparableConv2D(filters, kernel_size, strides=strides, name='separable_conv_1_%s' % id, + padding='same', use_bias=False, kernel_initializer='he_normal')(x) + x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name="separable_conv_1_bn_%s" % (id))(x) + x = Activation('relu')(x) + x = SeparableConv2D(filters, kernel_size, name='separable_conv_2_%s' % id, + padding='same', use_bias=False, kernel_initializer='he_normal')(x) + x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name="separable_conv_2_bn_%s" % (id))(x) return x @@ -446,26 +448,28 @@ def _adjust_block(p, ip, filters, id=None): p = ip elif p._keras_shape[img_dim] != ip._keras_shape[img_dim]: - p = Activation('relu', name='adjust_relu_1_%s' % id)(p) + with K.name_scope('adjust_reduction_block_%s' % id): + p = Activation('relu', name='adjust_relu_1_%s' % id)(p) - p1 = AveragePooling2D((1, 1), strides=(2, 2), padding='valid', name='adjust_avg_pool_1_%s' % id)(p) - p1 = Conv2D(filters // 2, (1, 1), padding='same', use_bias=False, - name='adjust_conv_1_%s' % id, kernel_initializer='he_normal')(p1) + p1 = AveragePooling2D((1, 1), strides=(2, 2), padding='valid', name='adjust_avg_pool_1_%s' % id)(p) + p1 = Conv2D(filters // 2, (1, 1), padding='same', use_bias=False, + name='adjust_conv_1_%s' % id, kernel_initializer='he_normal')(p1) - p2 = AveragePooling2D((1, 1), strides=(2, 2), padding='valid', name='adjust_avg_pool_2_%s' % id)(p) - p2 = Conv2D(filters // 2, (1, 1), padding='same', use_bias=False, - name='adjust_conv_2_%s' % id, kernel_initializer='he_normal')(p2) + p2 = AveragePooling2D((1, 1), strides=(2, 2), padding='valid', name='adjust_avg_pool_2_%s' % id)(p) + p2 = Conv2D(filters // 2, (1, 1), padding='same', use_bias=False, + name='adjust_conv_2_%s' % id, kernel_initializer='he_normal')(p2) - p = concatenate([p1, p2], axis=channel_dim) - p = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, - name='adjust_bn_%s' % id)(p) + p = concatenate([p1, p2], axis=channel_dim) + p = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name='adjust_bn_%s' % id)(p) elif p._keras_shape[channel_dim] != filters: - p = Activation('relu')(p) - p = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='adjust_conv_projection_%s' % id, - use_bias=False, kernel_initializer='he_normal')(p) - p = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, - name='adjust_bn_%s' % id)(p) + with K.name_scope('adjust_projection_block_%s' % id): + p = Activation('relu')(p) + p = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='adjust_conv_projection_%s' % id, + use_bias=False, kernel_initializer='he_normal')(p) + p = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name='adjust_bn_%s' % id)(p) return p @@ -491,23 +495,28 @@ def _normal_A(ip, p, filters, id=None): h = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, name='normal_bn_1_%s' % id)(h) - x1 = _separable_conv_block(h, filters, id='normal_left1_%s' % id) - x1 = add([x1, h], name='normal_add_1_%s' % id) + with K.name_scope('normal_A_block_1'): + x1 = _separable_conv_block(h, filters, id='normal_left1_%s' % id) + x1 = add([x1, h], name='normal_add_1_%s' % id) - x2_1 = _separable_conv_block(p, filters, id='normal_left2_%s' % id) - x2_2 = _separable_conv_block(h, filters, kernel_size=(5, 5), id='normal_right2_%s' % id) - x2 = add([x2_1, x2_2], name='normal_add_2_%s' % id) + with K.name_scope('normal_A_block_2'): + x2_1 = _separable_conv_block(p, filters, id='normal_left2_%s' % id) + x2_2 = _separable_conv_block(h, filters, kernel_size=(5, 5), id='normal_right2_%s' % id) + x2 = add([x2_1, x2_2], name='normal_add_2_%s' % id) - x3 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_left3_%s' % (id))(h) - x3 = add([x3, p], name='normal_add_3_%s' % id) + with K.name_scope('normal_A_block_3'): + x3 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_left3_%s' % (id))(h) + x3 = add([x3, p], name='normal_add_3_%s' % id) - x4_1 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_left4_%s' % (id))(p) - x4_2 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_right4_%s' % (id))(p) - x4 = add([x4_1, x4_2], name='normal_add_4_%s' % id) + with K.name_scope('normal_A_block_4'): + x4_1 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_left4_%s' % (id))(p) + x4_2 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_right4_%s' % (id))(p) + x4 = add([x4_1, x4_2], name='normal_add_4_%s' % id) - x5_1 = _separable_conv_block(p, filters, (5, 5), id='normal_left5_%s' % id) - x5_2 = _separable_conv_block(p, filters, (3, 3), id='normal_right5_%s' % id) - x5 = add([x5_1, x5_2], name='normal_add_5_%s' % id) + with K.name_scope('normal_A_block_5'): + x5_1 = _separable_conv_block(p, filters, (5, 5), id='normal_left5_%s' % id) + x5_2 = _separable_conv_block(p, filters, (3, 3), id='normal_right5_%s' % id) + x5 = add([x5_1, x5_2], name='normal_add_5_%s' % id) x = concatenate([p, x2, x5, x3, x4, x1], axis=channel_dim, name='normal_concat_%s' % id) return x, ip @@ -536,23 +545,28 @@ def _reduction_A(ip, p, filters, id=None): h = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, name='reduction_bn_1_%s' % id)(h) - x1_1 = _separable_conv_block(p, filters, (7, 7), strides=(2, 2), id='reduction_left1_%s' % id) - x1_2 = _separable_conv_block(h, filters, (5, 5), strides=(2, 2), id='reduction_right1_%s' % id) - x1 = add([x1_1, x1_2], name='reduction_add_1_%s' % id) + with K.name_scope('reduction_A_block_1'): + x1_1 = _separable_conv_block(p, filters, (7, 7), strides=(2, 2), id='reduction_left1_%s' % id) + x1_2 = _separable_conv_block(h, filters, (5, 5), strides=(2, 2), id='reduction_right1_%s' % id) + x1 = add([x1_1, x1_2], name='reduction_add_1_%s' % id) - x2_1 = MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left2_%s' % id)(h) - x2_2 = _separable_conv_block(p, filters, (7, 7), strides=(2, 2), id='reduction_right2_%s' % id) - x2 = add([x2_1, x2_2], name='reduction_add_2_%s' % id) + with K.name_scope('reduction_A_block_2'): + x2_1 = MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left2_%s' % id)(h) + x2_2 = _separable_conv_block(p, filters, (7, 7), strides=(2, 2), id='reduction_right2_%s' % id) + x2 = add([x2_1, x2_2], name='reduction_add_2_%s' % id) - x3_1 = AveragePooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left3_%s' % id)(h) - x3_2 = _separable_conv_block(p, filters, (5, 5), strides=(2, 2), id='reduction_right3_%s' % id) - x3 = add([x3_1, x3_2], name='reduction_add3_%s' % id) + with K.name_scope('reduction_A_block_3'): + x3_1 = AveragePooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left3_%s' % id)(h) + x3_2 = _separable_conv_block(p, filters, (5, 5), strides=(2, 2), id='reduction_right3_%s' % id) + x3 = add([x3_1, x3_2], name='reduction_add3_%s' % id) - x4_1 = MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left4_%s' % id)(h) - x4_2 = _separable_conv_block(x1, filters, (3, 3), id='reduction_right4_%s' % id) - x4 = add([x4_1, x4_2], name='reduction_add4_%s' % id) + with K.name_scope('reduction_A_block_4'): + x4_1 = MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left4_%s' % id)(h) + x4_2 = _separable_conv_block(x1, filters, (3, 3), id='reduction_right4_%s' % id) + x4 = add([x4_1, x4_2], name='reduction_add4_%s' % id) - x5 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='reduction_left5_%s' % id)(x1) + with K.name_scope('reduction_A_block_5'): + x5 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='reduction_left5_%s' % id)(x1) x = concatenate([x2, x3, x5, x4], axis=channel_dim, name='reduction_concat_%s' % id) return x, ip From a32ae325a8a7bbc52292f50943ae89314e664875 Mon Sep 17 00:00:00 2001 From: Somshubra Majumdar Date: Fri, 17 Nov 2017 12:47:06 -0600 Subject: [PATCH 19/29] Improve name scope usage --- keras_contrib/applications/nasnet.py | 181 ++++++++++++++------------- 1 file changed, 93 insertions(+), 88 deletions(-) diff --git a/keras_contrib/applications/nasnet.py b/keras_contrib/applications/nasnet.py index c7f52f8..ef37b79 100644 --- a/keras_contrib/applications/nasnet.py +++ b/keras_contrib/applications/nasnet.py @@ -52,7 +52,7 @@ def NASNet(input_shape=None, filters_multiplier=2, dropout=0.5, include_top=True, - weights='imagenet', + weights=None, input_tensor=None, pooling=None, classes=1000, @@ -170,7 +170,8 @@ def NASNet(input_shape=None, else: img_input = input_tensor - assert penultimate_filters % 24 == 0, "`penultimate_filters` needs to be divisible by 24" + assert penultimate_filters % 24 == 0, "`penultimate_filters` needs to be divisible " \ + "by 6 * (2^N)." channel_dim = 1 if K.image_data_format() == 'channels_first' else -1 filters = penultimate_filters // 24 @@ -197,22 +198,23 @@ def NASNet(input_shape=None, if use_auxilary_branch: img_dim = 2 if K.image_data_format() == 'channels_first' else -2 - auxilary_x = Activation('relu')(x) - auxilary_x = AveragePooling2D((5, 5), strides=(3, 3), padding='valid', name='aux_pool')(auxilary_x) - auxilary_x = Conv2D(128, (1, 1), padding='same', use_bias=False, name='aux_conv_projection', - kernel_initializer='he_normal')(auxilary_x) - auxilary_x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, - name='aux_bn_projection')(auxilary_x) - auxilary_x = Activation('relu')(auxilary_x) + with K.name_scope('auxilary_branch'): + auxilary_x = Activation('relu')(x) + auxilary_x = AveragePooling2D((5, 5), strides=(3, 3), padding='valid', name='aux_pool')(auxilary_x) + auxilary_x = Conv2D(128, (1, 1), padding='same', use_bias=False, name='aux_conv_projection', + kernel_initializer='he_normal')(auxilary_x) + auxilary_x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name='aux_bn_projection')(auxilary_x) + auxilary_x = Activation('relu')(auxilary_x) - auxilary_x = Conv2D(768, auxilary_x._keras_shape[img_dim], padding='valid', use_bias=False, - kernel_initializer='he_normal', name='aux_conv_reduction')(auxilary_x) - auxilary_x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, - name='aux_bn_reduction')(auxilary_x) - auxilary_x = Activation('relu')(auxilary_x) + auxilary_x = Conv2D(768, auxilary_x._keras_shape[img_dim], padding='valid', use_bias=False, + kernel_initializer='he_normal', name='aux_conv_reduction')(auxilary_x) + auxilary_x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name='aux_bn_reduction')(auxilary_x) + auxilary_x = Activation('relu')(auxilary_x) - auxilary_x = GlobalAveragePooling2D()(auxilary_x) - auxilary_x = Dense(classes, activation='softmax')(auxilary_x) + auxilary_x = GlobalAveragePooling2D()(auxilary_x) + auxilary_x = Dense(classes, activation='softmax')(auxilary_x) x, p0 = _reduction_A(x, p, filters * filters_multiplier ** 2, id='reduce_%d' % (2 * nb_blocks)) @@ -444,32 +446,33 @@ def _adjust_block(p, ip, filters, id=None): channel_dim = 1 if K.image_data_format() == 'channels_first' else -1 img_dim = 2 if K.image_data_format() == 'channels_first' else -2 - if p is None: - p = ip + with K.name_scope('adjust_block'): + if p is None: + p = ip - elif p._keras_shape[img_dim] != ip._keras_shape[img_dim]: - with K.name_scope('adjust_reduction_block_%s' % id): - p = Activation('relu', name='adjust_relu_1_%s' % id)(p) + elif p._keras_shape[img_dim] != ip._keras_shape[img_dim]: + with K.name_scope('adjust_reduction_block_%s' % id): + p = Activation('relu', name='adjust_relu_1_%s' % id)(p) - p1 = AveragePooling2D((1, 1), strides=(2, 2), padding='valid', name='adjust_avg_pool_1_%s' % id)(p) - p1 = Conv2D(filters // 2, (1, 1), padding='same', use_bias=False, - name='adjust_conv_1_%s' % id, kernel_initializer='he_normal')(p1) + p1 = AveragePooling2D((1, 1), strides=(2, 2), padding='valid', name='adjust_avg_pool_1_%s' % id)(p) + p1 = Conv2D(filters // 2, (1, 1), padding='same', use_bias=False, + name='adjust_conv_1_%s' % id, kernel_initializer='he_normal')(p1) - p2 = AveragePooling2D((1, 1), strides=(2, 2), padding='valid', name='adjust_avg_pool_2_%s' % id)(p) - p2 = Conv2D(filters // 2, (1, 1), padding='same', use_bias=False, - name='adjust_conv_2_%s' % id, kernel_initializer='he_normal')(p2) + p2 = AveragePooling2D((1, 1), strides=(2, 2), padding='valid', name='adjust_avg_pool_2_%s' % id)(p) + p2 = Conv2D(filters // 2, (1, 1), padding='same', use_bias=False, + name='adjust_conv_2_%s' % id, kernel_initializer='he_normal')(p2) - p = concatenate([p1, p2], axis=channel_dim) - p = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, - name='adjust_bn_%s' % id)(p) + p = concatenate([p1, p2], axis=channel_dim) + p = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name='adjust_bn_%s' % id)(p) - elif p._keras_shape[channel_dim] != filters: - with K.name_scope('adjust_projection_block_%s' % id): - p = Activation('relu')(p) - p = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='adjust_conv_projection_%s' % id, - use_bias=False, kernel_initializer='he_normal')(p) - p = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, - name='adjust_bn_%s' % id)(p) + elif p._keras_shape[channel_dim] != filters: + with K.name_scope('adjust_projection_block_%s' % id): + p = Activation('relu')(p) + p = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='adjust_conv_projection_%s' % id, + use_bias=False, kernel_initializer='he_normal')(p) + p = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name='adjust_bn_%s' % id)(p) return p @@ -487,38 +490,39 @@ def _normal_A(ip, p, filters, id=None): ''' channel_dim = 1 if K.image_data_format() == 'channels_first' else -1 - p = _adjust_block(p, ip, filters, id) + with K.name_scope('normal_A_block_%s' % id): + p = _adjust_block(p, ip, filters, id) - h = Activation('relu')(ip) - h = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='normal_conv_1_%s' % id, - use_bias=False, kernel_initializer='he_normal')(h) - h = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, - name='normal_bn_1_%s' % id)(h) + h = Activation('relu')(ip) + h = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='normal_conv_1_%s' % id, + use_bias=False, kernel_initializer='he_normal')(h) + h = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name='normal_bn_1_%s' % id)(h) - with K.name_scope('normal_A_block_1'): - x1 = _separable_conv_block(h, filters, id='normal_left1_%s' % id) - x1 = add([x1, h], name='normal_add_1_%s' % id) + with K.name_scope('block_1'): + x1 = _separable_conv_block(h, filters, id='normal_left1_%s' % id) + x1 = add([x1, h], name='normal_add_1_%s' % id) - with K.name_scope('normal_A_block_2'): - x2_1 = _separable_conv_block(p, filters, id='normal_left2_%s' % id) - x2_2 = _separable_conv_block(h, filters, kernel_size=(5, 5), id='normal_right2_%s' % id) - x2 = add([x2_1, x2_2], name='normal_add_2_%s' % id) + with K.name_scope('block_2'): + x2_1 = _separable_conv_block(p, filters, id='normal_left2_%s' % id) + x2_2 = _separable_conv_block(h, filters, kernel_size=(5, 5), id='normal_right2_%s' % id) + x2 = add([x2_1, x2_2], name='normal_add_2_%s' % id) - with K.name_scope('normal_A_block_3'): - x3 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_left3_%s' % (id))(h) - x3 = add([x3, p], name='normal_add_3_%s' % id) + with K.name_scope('block_3'): + x3 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_left3_%s' % (id))(h) + x3 = add([x3, p], name='normal_add_3_%s' % id) - with K.name_scope('normal_A_block_4'): - x4_1 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_left4_%s' % (id))(p) - x4_2 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_right4_%s' % (id))(p) - x4 = add([x4_1, x4_2], name='normal_add_4_%s' % id) + with K.name_scope('block_4'): + x4_1 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_left4_%s' % (id))(p) + x4_2 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_right4_%s' % (id))(p) + x4 = add([x4_1, x4_2], name='normal_add_4_%s' % id) - with K.name_scope('normal_A_block_5'): - x5_1 = _separable_conv_block(p, filters, (5, 5), id='normal_left5_%s' % id) - x5_2 = _separable_conv_block(p, filters, (3, 3), id='normal_right5_%s' % id) - x5 = add([x5_1, x5_2], name='normal_add_5_%s' % id) + with K.name_scope('block_5'): + x5_1 = _separable_conv_block(p, filters, (5, 5), id='normal_left5_%s' % id) + x5_2 = _separable_conv_block(p, filters, (3, 3), id='normal_right5_%s' % id) + x5 = add([x5_1, x5_2], name='normal_add_5_%s' % id) - x = concatenate([p, x2, x5, x3, x4, x1], axis=channel_dim, name='normal_concat_%s' % id) + x = concatenate([p, x2, x5, x3, x4, x1], axis=channel_dim, name='normal_concat_%s' % id) return x, ip @@ -537,36 +541,37 @@ def _reduction_A(ip, p, filters, id=None): """""" channel_dim = 1 if K.image_data_format() == 'channels_first' else -1 - p = _adjust_block(p, ip, filters, id) + with K.name_scope('reduction_A_block_%s' % id): + p = _adjust_block(p, ip, filters, id) - h = Activation('relu')(ip) - h = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='reduction_conv_1_%s' % id, - use_bias=False, kernel_initializer='he_normal')(h) - h = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, - name='reduction_bn_1_%s' % id)(h) + h = Activation('relu')(ip) + h = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='reduction_conv_1_%s' % id, + use_bias=False, kernel_initializer='he_normal')(h) + h = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name='reduction_bn_1_%s' % id)(h) - with K.name_scope('reduction_A_block_1'): - x1_1 = _separable_conv_block(p, filters, (7, 7), strides=(2, 2), id='reduction_left1_%s' % id) - x1_2 = _separable_conv_block(h, filters, (5, 5), strides=(2, 2), id='reduction_right1_%s' % id) - x1 = add([x1_1, x1_2], name='reduction_add_1_%s' % id) + with K.name_scope('block_1'): + x1_1 = _separable_conv_block(p, filters, (7, 7), strides=(2, 2), id='reduction_left1_%s' % id) + x1_2 = _separable_conv_block(h, filters, (5, 5), strides=(2, 2), id='reduction_right1_%s' % id) + x1 = add([x1_1, x1_2], name='reduction_add_1_%s' % id) - with K.name_scope('reduction_A_block_2'): - x2_1 = MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left2_%s' % id)(h) - x2_2 = _separable_conv_block(p, filters, (7, 7), strides=(2, 2), id='reduction_right2_%s' % id) - x2 = add([x2_1, x2_2], name='reduction_add_2_%s' % id) + with K.name_scope('block_2'): + x2_1 = MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left2_%s' % id)(h) + x2_2 = _separable_conv_block(p, filters, (7, 7), strides=(2, 2), id='reduction_right2_%s' % id) + x2 = add([x2_1, x2_2], name='reduction_add_2_%s' % id) - with K.name_scope('reduction_A_block_3'): - x3_1 = AveragePooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left3_%s' % id)(h) - x3_2 = _separable_conv_block(p, filters, (5, 5), strides=(2, 2), id='reduction_right3_%s' % id) - x3 = add([x3_1, x3_2], name='reduction_add3_%s' % id) + with K.name_scope('block_3'): + x3_1 = AveragePooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left3_%s' % id)(h) + x3_2 = _separable_conv_block(p, filters, (5, 5), strides=(2, 2), id='reduction_right3_%s' % id) + x3 = add([x3_1, x3_2], name='reduction_add3_%s' % id) - with K.name_scope('reduction_A_block_4'): - x4_1 = MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left4_%s' % id)(h) - x4_2 = _separable_conv_block(x1, filters, (3, 3), id='reduction_right4_%s' % id) - x4 = add([x4_1, x4_2], name='reduction_add4_%s' % id) + with K.name_scope('block_4'): + x4_1 = MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left4_%s' % id)(h) + x4_2 = _separable_conv_block(x1, filters, (3, 3), id='reduction_right4_%s' % id) + x4 = add([x4_1, x4_2], name='reduction_add4_%s' % id) - with K.name_scope('reduction_A_block_5'): - x5 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='reduction_left5_%s' % id)(x1) + with K.name_scope('block_5'): + x5 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='reduction_left5_%s' % id)(x1) - x = concatenate([x2, x3, x5, x4], axis=channel_dim, name='reduction_concat_%s' % id) - return x, ip + x = concatenate([x2, x3, x5, x4], axis=channel_dim, name='reduction_concat_%s' % id) + return x, ip From 0d9934b0bf9e10219c5a8d911b618cc936a60587 Mon Sep 17 00:00:00 2001 From: Somshubra Majumdar Date: Fri, 17 Nov 2017 13:06:30 -0600 Subject: [PATCH 20/29] Improve name scope and add CIFAR model --- keras_contrib/applications/nasnet.py | 73 +++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/keras_contrib/applications/nasnet.py b/keras_contrib/applications/nasnet.py index ef37b79..2ba3e4c 100644 --- a/keras_contrib/applications/nasnet.py +++ b/keras_contrib/applications/nasnet.py @@ -336,7 +336,7 @@ def NASNetMobile(input_shape=None, input_tensor=None, pooling=None, classes=1000): - """Instantiates a NASNet architecture in CIFAR mode. + """Instantiates a NASNet architecture in Mobile ImageNet mode. Note that only TensorFlow is supported for now, therefore it only works with the data format `image_data_format='channels_last'` in your Keras config @@ -399,6 +399,77 @@ def NASNetMobile(input_shape=None, default_size=224) +def NASNetCIFAR(input_shape=None, + dropout=0.0, + use_auxilary_branch=False, + include_top=True, + weights=None, + input_tensor=None, + pooling=None, + classes=10): + """Instantiates a NASNet architecture in CIFAR mode. + Note that only TensorFlow is supported for now, + therefore it only works with the data format + `image_data_format='channels_last'` in your Keras config + at `~/.keras/keras.json`. + + # Arguments + input_shape: optional shape tuple, only to be specified + if `include_top` is False (otherwise the input shape + has to be `(32, 32, 3)` for NASNetMobile + It should have exactly 3 inputs channels, + and width and height should be no smaller than 32. + E.g. `(32, 32, 3)` would be one valid value. + use_auxilary_branch: Whether to use the auxilary branch during + training or evaluation. + dropout: dropout rate + include_top: whether to include the fully-connected + layer at the top of the network. + weights: `None` (random initialization) or + `imagenet` (ImageNet weights) + input_tensor: optional Keras tensor (i.e. output of + `layers.Input()`) + to use as image input for the model. + pooling: Optional pooling mode for feature extraction + when `include_top` is `False`. + - `None` means that the output of the model + will be the 4D tensor output of the + last convolutional layer. + - `avg` means that global average pooling + will be applied to the output of the + last convolutional layer, and thus + the output of the model will be a + 2D tensor. + - `max` means that global max pooling will + be applied. + classes: optional number of classes to classify images + into, only to be specified if `include_top` is True, and + if no `weights` argument is specified. + default_size: specifies the default image size of the model + # Returns + A Keras model instance. + # Raises + ValueError: in case of invalid argument for `weights`, + or invalid input shape. + RuntimeError: If attempting to run this model with a + backend that does not support separable convolutions. + """ + return NASNet(input_shape, + penultimate_filters=768, + nb_blocks=2, + stem_filters=96, + skip_reduction=True, + use_auxilary_branch=use_auxilary_branch, + filters_multiplier=2, + dropout=dropout, + include_top=include_top, + weights=weights, + input_tensor=input_tensor, + pooling=pooling, + classes=classes, + default_size=224) + + def _separable_conv_block(ip, filters, kernel_size=(3, 3), strides=(1, 1), id=None): '''Adds 2 blocks of [relu-separable conv-batchnorm] From 4a99de40a24aec5ff860cebd5f53c89e26dc06ac Mon Sep 17 00:00:00 2001 From: Somshubra Majumdar Date: Fri, 17 Nov 2017 15:41:30 -0600 Subject: [PATCH 21/29] Improve name scope and add CIFAR model --- keras_contrib/applications/nasnet.py | 179 +++++++++++++-------------- 1 file changed, 89 insertions(+), 90 deletions(-) diff --git a/keras_contrib/applications/nasnet.py b/keras_contrib/applications/nasnet.py index 2ba3e4c..15e7247 100644 --- a/keras_contrib/applications/nasnet.py +++ b/keras_contrib/applications/nasnet.py @@ -38,7 +38,6 @@ from keras.applications.inception_v3 import preprocess_input from keras.applications.imagenet_utils import decode_predictions from keras import backend as K - _BN_DECAY = 0.9997 _BN_EPSILON = 1e-3 @@ -171,10 +170,10 @@ def NASNet(input_shape=None, img_input = input_tensor assert penultimate_filters % 24 == 0, "`penultimate_filters` needs to be divisible " \ - "by 6 * (2^N)." + "by 6 * (2^N)." channel_dim = 1 if K.image_data_format() == 'channels_first' else -1 - filters = penultimate_filters // 24 + filters = penultimate_filters // 24 x = Conv2D(stem_filters, (3, 3), strides=(2, 2), padding='valid', use_bias=False, name='stem_conv1', kernel_initializer='he_normal')(img_input) @@ -199,22 +198,22 @@ def NASNet(input_shape=None, img_dim = 2 if K.image_data_format() == 'channels_first' else -2 with K.name_scope('auxilary_branch'): - auxilary_x = Activation('relu')(x) - auxilary_x = AveragePooling2D((5, 5), strides=(3, 3), padding='valid', name='aux_pool')(auxilary_x) - auxilary_x = Conv2D(128, (1, 1), padding='same', use_bias=False, name='aux_conv_projection', - kernel_initializer='he_normal')(auxilary_x) - auxilary_x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, - name='aux_bn_projection')(auxilary_x) - auxilary_x = Activation('relu')(auxilary_x) + auxilary_x = Activation('relu')(x) + auxilary_x = AveragePooling2D((5, 5), strides=(3, 3), padding='valid', name='aux_pool')(auxilary_x) + auxilary_x = Conv2D(128, (1, 1), padding='same', use_bias=False, name='aux_conv_projection', + kernel_initializer='he_normal')(auxilary_x) + auxilary_x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name='aux_bn_projection')(auxilary_x) + auxilary_x = Activation('relu')(auxilary_x) - auxilary_x = Conv2D(768, auxilary_x._keras_shape[img_dim], padding='valid', use_bias=False, - kernel_initializer='he_normal', name='aux_conv_reduction')(auxilary_x) - auxilary_x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, - name='aux_bn_reduction')(auxilary_x) - auxilary_x = Activation('relu')(auxilary_x) + auxilary_x = Conv2D(768, auxilary_x._keras_shape[img_dim], padding='valid', use_bias=False, + kernel_initializer='he_normal', name='aux_conv_reduction')(auxilary_x) + auxilary_x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name='aux_bn_reduction')(auxilary_x) + auxilary_x = Activation('relu')(auxilary_x) - auxilary_x = GlobalAveragePooling2D()(auxilary_x) - auxilary_x = Dense(classes, activation='softmax')(auxilary_x) + auxilary_x = GlobalAveragePooling2D()(auxilary_x) + auxilary_x = Dense(classes, activation='softmax')(auxilary_x) x, p0 = _reduction_A(x, p, filters * filters_multiplier ** 2, id='reduce_%d' % (2 * nb_blocks)) @@ -456,7 +455,7 @@ def NASNetCIFAR(input_shape=None, """ return NASNet(input_shape, penultimate_filters=768, - nb_blocks=2, + nb_blocks=6, stem_filters=96, skip_reduction=True, use_auxilary_branch=use_auxilary_branch, @@ -518,32 +517,32 @@ def _adjust_block(p, ip, filters, id=None): img_dim = 2 if K.image_data_format() == 'channels_first' else -2 with K.name_scope('adjust_block'): - if p is None: - p = ip + if p is None: + p = ip - elif p._keras_shape[img_dim] != ip._keras_shape[img_dim]: - with K.name_scope('adjust_reduction_block_%s' % id): - p = Activation('relu', name='adjust_relu_1_%s' % id)(p) + elif p._keras_shape[img_dim] != ip._keras_shape[img_dim]: + with K.name_scope('adjust_reduction_block_%s' % id): + p = Activation('relu', name='adjust_relu_1_%s' % id)(p) - p1 = AveragePooling2D((1, 1), strides=(2, 2), padding='valid', name='adjust_avg_pool_1_%s' % id)(p) - p1 = Conv2D(filters // 2, (1, 1), padding='same', use_bias=False, - name='adjust_conv_1_%s' % id, kernel_initializer='he_normal')(p1) + p1 = AveragePooling2D((1, 1), strides=(2, 2), padding='valid', name='adjust_avg_pool_1_%s' % id)(p) + p1 = Conv2D(filters // 2, (1, 1), padding='same', use_bias=False, + name='adjust_conv_1_%s' % id, kernel_initializer='he_normal')(p1) - p2 = AveragePooling2D((1, 1), strides=(2, 2), padding='valid', name='adjust_avg_pool_2_%s' % id)(p) - p2 = Conv2D(filters // 2, (1, 1), padding='same', use_bias=False, - name='adjust_conv_2_%s' % id, kernel_initializer='he_normal')(p2) + p2 = AveragePooling2D((1, 1), strides=(2, 2), padding='valid', name='adjust_avg_pool_2_%s' % id)(p) + p2 = Conv2D(filters // 2, (1, 1), padding='same', use_bias=False, + name='adjust_conv_2_%s' % id, kernel_initializer='he_normal')(p2) - p = concatenate([p1, p2], axis=channel_dim) - p = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, - name='adjust_bn_%s' % id)(p) + p = concatenate([p1, p2], axis=channel_dim) + p = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name='adjust_bn_%s' % id)(p) - elif p._keras_shape[channel_dim] != filters: - with K.name_scope('adjust_projection_block_%s' % id): - p = Activation('relu')(p) - p = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='adjust_conv_projection_%s' % id, - use_bias=False, kernel_initializer='he_normal')(p) - p = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, - name='adjust_bn_%s' % id)(p) + elif p._keras_shape[channel_dim] != filters: + with K.name_scope('adjust_projection_block_%s' % id): + p = Activation('relu')(p) + p = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='adjust_conv_projection_%s' % id, + use_bias=False, kernel_initializer='he_normal')(p) + p = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name='adjust_bn_%s' % id)(p) return p @@ -562,38 +561,38 @@ def _normal_A(ip, p, filters, id=None): channel_dim = 1 if K.image_data_format() == 'channels_first' else -1 with K.name_scope('normal_A_block_%s' % id): - p = _adjust_block(p, ip, filters, id) + p = _adjust_block(p, ip, filters, id) - h = Activation('relu')(ip) - h = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='normal_conv_1_%s' % id, - use_bias=False, kernel_initializer='he_normal')(h) - h = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, - name='normal_bn_1_%s' % id)(h) + h = Activation('relu')(ip) + h = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='normal_conv_1_%s' % id, + use_bias=False, kernel_initializer='he_normal')(h) + h = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name='normal_bn_1_%s' % id)(h) - with K.name_scope('block_1'): - x1 = _separable_conv_block(h, filters, id='normal_left1_%s' % id) - x1 = add([x1, h], name='normal_add_1_%s' % id) + with K.name_scope('block_1'): + x1 = _separable_conv_block(h, filters, id='normal_left1_%s' % id) + x1 = add([x1, h], name='normal_add_1_%s' % id) - with K.name_scope('block_2'): - x2_1 = _separable_conv_block(p, filters, id='normal_left2_%s' % id) - x2_2 = _separable_conv_block(h, filters, kernel_size=(5, 5), id='normal_right2_%s' % id) - x2 = add([x2_1, x2_2], name='normal_add_2_%s' % id) + with K.name_scope('block_2'): + x2_1 = _separable_conv_block(p, filters, id='normal_left2_%s' % id) + x2_2 = _separable_conv_block(h, filters, kernel_size=(5, 5), id='normal_right2_%s' % id) + x2 = add([x2_1, x2_2], name='normal_add_2_%s' % id) - with K.name_scope('block_3'): - x3 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_left3_%s' % (id))(h) - x3 = add([x3, p], name='normal_add_3_%s' % id) + with K.name_scope('block_3'): + x3 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_left3_%s' % (id))(h) + x3 = add([x3, p], name='normal_add_3_%s' % id) - with K.name_scope('block_4'): - x4_1 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_left4_%s' % (id))(p) - x4_2 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_right4_%s' % (id))(p) - x4 = add([x4_1, x4_2], name='normal_add_4_%s' % id) + with K.name_scope('block_4'): + x4_1 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_left4_%s' % (id))(p) + x4_2 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_right4_%s' % (id))(p) + x4 = add([x4_1, x4_2], name='normal_add_4_%s' % id) - with K.name_scope('block_5'): - x5_1 = _separable_conv_block(p, filters, (5, 5), id='normal_left5_%s' % id) - x5_2 = _separable_conv_block(p, filters, (3, 3), id='normal_right5_%s' % id) - x5 = add([x5_1, x5_2], name='normal_add_5_%s' % id) + with K.name_scope('block_5'): + x5_1 = _separable_conv_block(p, filters, (5, 5), id='normal_left5_%s' % id) + x5_2 = _separable_conv_block(p, filters, (3, 3), id='normal_right5_%s' % id) + x5 = add([x5_1, x5_2], name='normal_add_5_%s' % id) - x = concatenate([p, x2, x5, x3, x4, x1], axis=channel_dim, name='normal_concat_%s' % id) + x = concatenate([p, x2, x5, x3, x4, x1], axis=channel_dim, name='normal_concat_%s' % id) return x, ip @@ -613,36 +612,36 @@ def _reduction_A(ip, p, filters, id=None): channel_dim = 1 if K.image_data_format() == 'channels_first' else -1 with K.name_scope('reduction_A_block_%s' % id): - p = _adjust_block(p, ip, filters, id) + p = _adjust_block(p, ip, filters, id) - h = Activation('relu')(ip) - h = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='reduction_conv_1_%s' % id, - use_bias=False, kernel_initializer='he_normal')(h) - h = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, - name='reduction_bn_1_%s' % id)(h) + h = Activation('relu')(ip) + h = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='reduction_conv_1_%s' % id, + use_bias=False, kernel_initializer='he_normal')(h) + h = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, + name='reduction_bn_1_%s' % id)(h) - with K.name_scope('block_1'): - x1_1 = _separable_conv_block(p, filters, (7, 7), strides=(2, 2), id='reduction_left1_%s' % id) - x1_2 = _separable_conv_block(h, filters, (5, 5), strides=(2, 2), id='reduction_right1_%s' % id) - x1 = add([x1_1, x1_2], name='reduction_add_1_%s' % id) + with K.name_scope('block_1'): + x1_1 = _separable_conv_block(p, filters, (7, 7), strides=(2, 2), id='reduction_left1_%s' % id) + x1_2 = _separable_conv_block(h, filters, (5, 5), strides=(2, 2), id='reduction_right1_%s' % id) + x1 = add([x1_1, x1_2], name='reduction_add_1_%s' % id) - with K.name_scope('block_2'): - x2_1 = MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left2_%s' % id)(h) - x2_2 = _separable_conv_block(p, filters, (7, 7), strides=(2, 2), id='reduction_right2_%s' % id) - x2 = add([x2_1, x2_2], name='reduction_add_2_%s' % id) + with K.name_scope('block_2'): + x2_1 = MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left2_%s' % id)(h) + x2_2 = _separable_conv_block(p, filters, (7, 7), strides=(2, 2), id='reduction_right2_%s' % id) + x2 = add([x2_1, x2_2], name='reduction_add_2_%s' % id) - with K.name_scope('block_3'): - x3_1 = AveragePooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left3_%s' % id)(h) - x3_2 = _separable_conv_block(p, filters, (5, 5), strides=(2, 2), id='reduction_right3_%s' % id) - x3 = add([x3_1, x3_2], name='reduction_add3_%s' % id) + with K.name_scope('block_3'): + x3_1 = AveragePooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left3_%s' % id)(h) + x3_2 = _separable_conv_block(p, filters, (5, 5), strides=(2, 2), id='reduction_right3_%s' % id) + x3 = add([x3_1, x3_2], name='reduction_add3_%s' % id) - with K.name_scope('block_4'): - x4_1 = MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left4_%s' % id)(h) - x4_2 = _separable_conv_block(x1, filters, (3, 3), id='reduction_right4_%s' % id) - x4 = add([x4_1, x4_2], name='reduction_add4_%s' % id) + with K.name_scope('block_4'): + x4_1 = MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left4_%s' % id)(h) + x4_2 = _separable_conv_block(x1, filters, (3, 3), id='reduction_right4_%s' % id) + x4 = add([x4_1, x4_2], name='reduction_add4_%s' % id) - with K.name_scope('block_5'): - x5 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='reduction_left5_%s' % id)(x1) + with K.name_scope('block_5'): + x5 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='reduction_left5_%s' % id)(x1) - x = concatenate([x2, x3, x5, x4], axis=channel_dim, name='reduction_concat_%s' % id) - return x, ip + x = concatenate([x2, x3, x5, x4], axis=channel_dim, name='reduction_concat_%s' % id) + return x, ip From 1d5fed1f0842eb90861f43762d69b67472e5ba5e Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sat, 18 Nov 2017 11:34:13 -0500 Subject: [PATCH 22/29] .travis.yml try fixing mkl errors --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 3564883..c550a23 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,7 @@ install: - source activate test-environment - pip install pytest-cov python-coveralls pytest-xdist coverage==3.7.1 #we need this version of coverage for coveralls.io to work - pip install pep8 pytest-pep8 + - pip install mkl mkl-service - pip install theano - pip install git+git://github.com/fchollet/keras.git From 052b05ad755373f8a6b7e0775d80b4226f13cd2a Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sat, 18 Nov 2017 12:04:21 -0500 Subject: [PATCH 23/29] .travis.yml second mkl fix attempt --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c550a23..f039ae3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,7 +34,7 @@ install: - source activate test-environment - pip install pytest-cov python-coveralls pytest-xdist coverage==3.7.1 #we need this version of coverage for coveralls.io to work - pip install pep8 pytest-pep8 - - pip install mkl mkl-service + - conda install mkl mkl-service - pip install theano - pip install git+git://github.com/fchollet/keras.git From 8f0d77448aece93d75544765435a7bd6c10cbb8a Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sat, 18 Nov 2017 13:17:49 -0500 Subject: [PATCH 24/29] .travis.yml set MKL_THREADING_LAYER="GNU" --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index f039ae3..39013c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,6 +64,7 @@ install: # command to run tests script: + - export MKL_THREADING_LAYER="GNU" # run keras backend init to initialize backend config - python -c "import keras.backend" # create dataset directory to avoid concurrent directory creation at runtime From 09ec44fa80fdd3604d2261370ab53319d8545b9e Mon Sep 17 00:00:00 2001 From: Somshubra Majumdar Date: Sat, 18 Nov 2017 13:38:12 -0600 Subject: [PATCH 25/29] Improve auxilary head --- keras_contrib/applications/nasnet.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/keras_contrib/applications/nasnet.py b/keras_contrib/applications/nasnet.py index 15e7247..cddc682 100644 --- a/keras_contrib/applications/nasnet.py +++ b/keras_contrib/applications/nasnet.py @@ -195,7 +195,8 @@ def NASNet(input_shape=None, auxilary_x = None if use_auxilary_branch: - img_dim = 2 if K.image_data_format() == 'channels_first' else -2 + img_height = 1 if K.image_data_format() == 'channels_first' else 2 + img_width = 2 if K.image_data_format() == 'channels_first' else 3 with K.name_scope('auxilary_branch'): auxilary_x = Activation('relu')(x) @@ -206,8 +207,9 @@ def NASNet(input_shape=None, name='aux_bn_projection')(auxilary_x) auxilary_x = Activation('relu')(auxilary_x) - auxilary_x = Conv2D(768, auxilary_x._keras_shape[img_dim], padding='valid', use_bias=False, - kernel_initializer='he_normal', name='aux_conv_reduction')(auxilary_x) + auxilary_x = Conv2D(768, (auxilary_x._keras_shape[img_height], auxilary_x._keras_shape[img_width]), + padding='valid', use_bias=False, kernel_initializer='he_normal', + name='aux_conv_reduction')(auxilary_x) auxilary_x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON, name='aux_bn_reduction')(auxilary_x) auxilary_x = Activation('relu')(auxilary_x) From 17ac57d02a480197f61254b1712d347ae7d052c0 Mon Sep 17 00:00:00 2001 From: Somshubra Majumdar Date: Sat, 18 Nov 2017 13:52:05 -0600 Subject: [PATCH 26/29] Add example script --- examples/cifar10_nasnet.py | 97 ++++++++++++++++++++++++++++ keras_contrib/applications/nasnet.py | 2 +- 2 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 examples/cifar10_nasnet.py diff --git a/examples/cifar10_nasnet.py b/examples/cifar10_nasnet.py new file mode 100644 index 0000000..8eee651 --- /dev/null +++ b/examples/cifar10_nasnet.py @@ -0,0 +1,97 @@ +""" +Adapted from keras example cifar10_cnn.py +Train NASNet-CIFAR on the CIFAR10 small images dataset. + +GPU run command with Theano backend (with TensorFlow, the GPU is automatically used): + THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python cifar10_nasnet.py +""" +from __future__ import print_function +from keras.datasets import cifar10 +from keras.preprocessing.image import ImageDataGenerator +from keras.utils import np_utils +from keras.callbacks import ModelCheckpoint +from keras.callbacks import ReduceLROnPlateau +from keras.callbacks import CSVLogger +from keras_contrib.applications.nasnet import NASNetCIFAR + +import numpy as np + + +weights_file = 'NASNet-CIFAR-10.h5' +lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.5), cooldown=0, patience=5, min_lr=0.5e-6) +csv_logger = CSVLogger('NASNet-CIFAR-10.csv') +model_checkpoint = ModelCheckpoint(weights_file, monitor='val_predictions_acc', save_best_only=True, + save_weights_only=True, mode='max') + +batch_size = 128 +nb_classes = 10 +nb_epoch = 200 +data_augmentation = True + +# input image dimensions +img_rows, img_cols = 32, 32 +# The CIFAR10 images are RGB. +img_channels = 3 + +# The data, shuffled and split between train and test sets: +(X_train, y_train), (X_test, y_test) = cifar10.load_data() + +# Convert class vectors to binary class matrices. +Y_train = np_utils.to_categorical(y_train, nb_classes) +Y_test = np_utils.to_categorical(y_test, nb_classes) + +X_train = X_train.astype('float32') +X_test = X_test.astype('float32') + +# subtract mean and normalize +mean_image = np.mean(X_train, axis=0) +X_train -= mean_image +X_test -= mean_image +X_train /= 128. +X_test /= 128. + +# For training, the auxilary branch must be used to correctly train NASNet +model = NASNetCIFAR((img_rows, img_cols, img_channels), dropout=0.5, + use_auxilary_branch=True) +model.compile(loss=['categorical_crossentropy', 'categorical_crossentropy'], + optimizer='adam', + loss_weights=[1.0, 0.4], + metrics=['accuracy']) + +if not data_augmentation: + print('Not using data augmentation.') + model.fit(X_train, Y_train, + batch_size=batch_size, + nb_epoch=nb_epoch, + validation_data=(X_test, Y_test), + shuffle=True, + callbacks=[lr_reducer, csv_logger, model_checkpoint]) +else: + print('Using real-time data augmentation.') + # This will do preprocessing and realtime data augmentation: + datagen = ImageDataGenerator( + featurewise_center=False, # set input mean to 0 over the dataset + samplewise_center=False, # set each sample mean to 0 + featurewise_std_normalization=False, # divide inputs by std of the dataset + samplewise_std_normalization=False, # divide each input by its std + zca_whitening=False, # apply ZCA whitening + rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180) + width_shift_range=0.1, # randomly shift images horizontally (fraction of total width) + height_shift_range=0.1, # randomly shift images vertically (fraction of total height) + horizontal_flip=True, # randomly flip images + vertical_flip=False) # randomly flip images + + # Compute quantities required for featurewise normalization + # (std, mean, and principal components if ZCA whitening is applied). + datagen.fit(X_train) + + # Fit the model on the batches generated by datagen.flow(). + model.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_size), + steps_per_epoch=X_train.shape[0] // batch_size, + validation_data=(X_test, Y_test), + epochs=nb_epoch, verbose=2, + callbacks=[lr_reducer, csv_logger, model_checkpoint]) + +scores = model.evaluate(X_test, Y_test, batch_size=batch_size) +for score, metric_name in zip(scores, model.metrics_names): + print("%s : %0.4f" % (metric_name, score)) diff --git a/keras_contrib/applications/nasnet.py b/keras_contrib/applications/nasnet.py index cddc682..072172d 100644 --- a/keras_contrib/applications/nasnet.py +++ b/keras_contrib/applications/nasnet.py @@ -215,7 +215,7 @@ def NASNet(input_shape=None, auxilary_x = Activation('relu')(auxilary_x) auxilary_x = GlobalAveragePooling2D()(auxilary_x) - auxilary_x = Dense(classes, activation='softmax')(auxilary_x) + auxilary_x = Dense(classes, activation='softmax', name='aux_predictions')(auxilary_x) x, p0 = _reduction_A(x, p, filters * filters_multiplier ** 2, id='reduce_%d' % (2 * nb_blocks)) From 68acf5e2eea5c8dac90f38c1e7782456ec61faa6 Mon Sep 17 00:00:00 2001 From: lameeus Date: Mon, 20 Nov 2017 11:46:24 +0100 Subject: [PATCH 27/29] added absolute tolerance to handle comparing zeros --- tests/keras_contrib/backend/backend_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/keras_contrib/backend/backend_test.py b/tests/keras_contrib/backend/backend_test.py index 998bc6b..1e4f344 100644 --- a/tests/keras_contrib/backend/backend_test.py +++ b/tests/keras_contrib/backend/backend_test.py @@ -157,8 +157,9 @@ class TestBackend(object): th_var_val = KTH.eval(th_var) tf_var_val = KTF.eval(tf_var) - assert_allclose(th_mean_val, tf_mean_val, rtol=1e-4) - assert_allclose(th_var_val, tf_var_val, rtol=1e-4) + # absolute tolerance needed when working with zeros + assert_allclose(th_mean_val, tf_mean_val, rtol=1e-4, atol=1e-10) + assert_allclose(th_var_val, tf_var_val, rtol=1e-4, atol=1e-10) def test_clip(self): check_single_tensor_operation('clip', (4, 2), min_value=0.4, max_value=0.6) From aeebb8c5204222d03f7e45013e1ca66b377a8f7b Mon Sep 17 00:00:00 2001 From: Somshubra Majumdar Date: Thu, 23 Nov 2017 13:57:21 -0600 Subject: [PATCH 28/29] Set default weights to None since ImageNet weights arent available yet --- keras_contrib/applications/nasnet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/keras_contrib/applications/nasnet.py b/keras_contrib/applications/nasnet.py index 072172d..34f2e11 100644 --- a/keras_contrib/applications/nasnet.py +++ b/keras_contrib/applications/nasnet.py @@ -262,7 +262,7 @@ def NASNetLarge(input_shape=None, dropout=0.5, use_auxilary_branch=False, include_top=True, - weights='imagenet', + weights=None, input_tensor=None, pooling=None, classes=1000): @@ -333,7 +333,7 @@ def NASNetMobile(input_shape=None, dropout=0.5, use_auxilary_branch=False, include_top=True, - weights='imagenet', + weights=None, input_tensor=None, pooling=None, classes=1000): From 3ab207d8a5d2c081a7be9fdb4188f18cf32109d4 Mon Sep 17 00:00:00 2001 From: Somshubra Majumdar Date: Wed, 29 Nov 2017 08:56:02 -0600 Subject: [PATCH 29/29] Fix https://github.com/titu1994/Keras-NASNet/issues/2 --- keras_contrib/applications/nasnet.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/keras_contrib/applications/nasnet.py b/keras_contrib/applications/nasnet.py index 34f2e11..84b7e80 100644 --- a/keras_contrib/applications/nasnet.py +++ b/keras_contrib/applications/nasnet.py @@ -29,6 +29,8 @@ from keras.layers import GlobalAveragePooling2D from keras.layers import GlobalMaxPooling2D from keras.layers import Conv2D from keras.layers import SeparableConv2D +from keras.layers import ZeroPadding2D +from keras.layers import Cropping2D from keras.layers import concatenate from keras.layers import add from keras.utils.data_utils import get_file @@ -262,7 +264,7 @@ def NASNetLarge(input_shape=None, dropout=0.5, use_auxilary_branch=False, include_top=True, - weights=None, + weights='imagenet', input_tensor=None, pooling=None, classes=1000): @@ -333,7 +335,7 @@ def NASNetMobile(input_shape=None, dropout=0.5, use_auxilary_branch=False, include_top=True, - weights=None, + weights='imagenet', input_tensor=None, pooling=None, classes=1000): @@ -530,7 +532,9 @@ def _adjust_block(p, ip, filters, id=None): p1 = Conv2D(filters // 2, (1, 1), padding='same', use_bias=False, name='adjust_conv_1_%s' % id, kernel_initializer='he_normal')(p1) - p2 = AveragePooling2D((1, 1), strides=(2, 2), padding='valid', name='adjust_avg_pool_2_%s' % id)(p) + p2 = ZeroPadding2D(padding=((0, 1), (0, 1)))(p) + p2 = Cropping2D(cropping=((1, 0), (1, 0)))(p2) + p2 = AveragePooling2D((1, 1), strides=(2, 2), padding='valid', name='adjust_avg_pool_2_%s' % id)(p2) p2 = Conv2D(filters // 2, (1, 1), padding='same', use_bias=False, name='adjust_conv_2_%s' % id, kernel_initializer='he_normal')(p2)