Merge remote-tracking branch 'upstream/master'

.travis.yml

@@ -34,6 +34,7 @@ install:
   - source activate test-environment
   - pip install pytest-cov python-coveralls pytest-xdist coverage==3.7.1 #we need this version of coverage for coveralls.io to work
   - pip install pep8 pytest-pep8
+  - conda install mkl mkl-service
   - pip install theano
   - pip install git+git://github.com/fchollet/keras.git
 
@@ -63,6 +64,7 @@ install:
 
 # command to run tests
 script:
+  - export MKL_THREADING_LAYER="GNU"
   # run keras backend init to initialize backend config
   - python -c "import keras.backend"
   # create dataset directory to avoid concurrent directory creation at runtime

examples/cifar10_nasnet.py

@@ -0,0 +1,97 @@
+"""
+Adapted from keras example cifar10_cnn.py
+Train NASNet-CIFAR on the CIFAR10 small images dataset.
+
+GPU run command with Theano backend (with TensorFlow, the GPU is automatically used):
+    THEANO_FLAGS=mode=FAST_RUN,device=gpu,floatX=float32 python cifar10_nasnet.py
+"""
+from __future__ import print_function
+from keras.datasets import cifar10
+from keras.preprocessing.image import ImageDataGenerator
+from keras.utils import np_utils
+from keras.callbacks import ModelCheckpoint
+from keras.callbacks import ReduceLROnPlateau
+from keras.callbacks import CSVLogger
+from keras_contrib.applications.nasnet import NASNetCIFAR
+
+import numpy as np
+
+
+weights_file = 'NASNet-CIFAR-10.h5'
+lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.5), cooldown=0, patience=5, min_lr=0.5e-6)
+csv_logger = CSVLogger('NASNet-CIFAR-10.csv')
+model_checkpoint = ModelCheckpoint(weights_file, monitor='val_predictions_acc', save_best_only=True,
+                                   save_weights_only=True, mode='max')
+
+batch_size = 128
+nb_classes = 10
+nb_epoch = 200
+data_augmentation = True
+
+# input image dimensions
+img_rows, img_cols = 32, 32
+# The CIFAR10 images are RGB.
+img_channels = 3
+
+# The data, shuffled and split between train and test sets:
+(X_train, y_train), (X_test, y_test) = cifar10.load_data()
+
+# Convert class vectors to binary class matrices.
+Y_train = np_utils.to_categorical(y_train, nb_classes)
+Y_test = np_utils.to_categorical(y_test, nb_classes)
+
+X_train = X_train.astype('float32')
+X_test = X_test.astype('float32')
+
+# subtract mean and normalize
+mean_image = np.mean(X_train, axis=0)
+X_train -= mean_image
+X_test -= mean_image
+X_train /= 128.
+X_test /= 128.
+
+# For training, the auxilary branch must be used to correctly train NASNet
+model = NASNetCIFAR((img_rows, img_cols, img_channels), dropout=0.5,
+                    use_auxilary_branch=True)
+model.compile(loss=['categorical_crossentropy', 'categorical_crossentropy'],
+              optimizer='adam',
+              loss_weights=[1.0, 0.4],
+              metrics=['accuracy'])
+
+if not data_augmentation:
+    print('Not using data augmentation.')
+    model.fit(X_train, Y_train,
+              batch_size=batch_size,
+              nb_epoch=nb_epoch,
+              validation_data=(X_test, Y_test),
+              shuffle=True,
+              callbacks=[lr_reducer, csv_logger, model_checkpoint])
+else:
+    print('Using real-time data augmentation.')
+    # This will do preprocessing and realtime data augmentation:
+    datagen = ImageDataGenerator(
+        featurewise_center=False,  # set input mean to 0 over the dataset
+        samplewise_center=False,  # set each sample mean to 0
+        featurewise_std_normalization=False,  # divide inputs by std of the dataset
+        samplewise_std_normalization=False,  # divide each input by its std
+        zca_whitening=False,  # apply ZCA whitening
+        rotation_range=0,  # randomly rotate images in the range (degrees, 0 to 180)
+        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)
+        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)
+        horizontal_flip=True,  # randomly flip images
+        vertical_flip=False)  # randomly flip images
+
+    # Compute quantities required for featurewise normalization
+    # (std, mean, and principal components if ZCA whitening is applied).
+    datagen.fit(X_train)
+
+    # Fit the model on the batches generated by datagen.flow().
+    model.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_size),
+                        steps_per_epoch=X_train.shape[0] // batch_size,
+                        validation_data=(X_test, Y_test),
+                        epochs=nb_epoch, verbose=2,
+                        callbacks=[lr_reducer, csv_logger, model_checkpoint])
+
+scores = model.evaluate(X_test, Y_test, batch_size=batch_size)
+for score, metric_name in zip(scores, model.metrics_names):
+    print("%s : %0.4f" % (metric_name, score))

keras_contrib/applications/__init__.py

@@ -1,2 +1,5 @@
 from .densenet import DenseNet
 from .ror import ResidualOfResidual
+from .resnet import ResNet, ResNet18, ResNet34, ResNet50, ResNet101, ResNet152
+from .wide_resnet import WideResidualNetwork
+from .nasnet import NASNet, NASNetLarge, NASNetMobile

keras_contrib/applications/densenet.py

@@ -506,7 +506,11 @@ def DenseNetImageNet161(input_shape=None,
                     pooling=pooling, classes=classes, activation=activation)
 
 
-def __conv_block(ip, nb_filter, bottleneck=False, dropout_rate=None, weight_decay=1e-4):
+def name_or_none(prefix, name):
+    return prefix + name if (prefix is not None and name is not None) else None
+
+
+def __conv_block(ip, nb_filter, bottleneck=False, dropout_rate=None, weight_decay=1e-4, block_prefix=None):
     '''
     Adds a convolution layer (with batch normalization and relu),
     and optionally a bottleneck layer.
@@ -518,6 +522,7 @@ def __conv_block(ip, nb_filter, bottleneck=False, dropout_rate=None, weight_deca
         bottleneck: if True, adds a bottleneck convolution block
         dropout_rate: dropout rate
         weight_decay: weight decay factor
+        block_prefix: str, for unique layer naming
 
      # Input shape
         4D tensor with shape:
@@ -538,18 +543,20 @@ def __conv_block(ip, nb_filter, bottleneck=False, dropout_rate=None, weight_deca
     with K.name_scope('ConvBlock'):
         concat_axis = 1 if K.image_data_format() == 'channels_first' else -1
 
-        x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(ip)
+        x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5, name=name_or_none(block_prefix, '_bn'))(ip)
         x = Activation('relu')(x)
 
         if bottleneck:
             inter_channel = nb_filter * 4
 
             x = Conv2D(inter_channel, (1, 1), kernel_initializer='he_normal', padding='same', use_bias=False,
-                       kernel_regularizer=l2(weight_decay))(x)
-            x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(x)
+                       kernel_regularizer=l2(weight_decay), name=name_or_none(block_prefix, '_bottleneck_conv2D'))(x)
+            x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5,
+                                   name=name_or_none(block_prefix, '_bottleneck_bn'))(x)
             x = Activation('relu')(x)
 
-        x = Conv2D(nb_filter, (3, 3), kernel_initializer='he_normal', padding='same', use_bias=False)(x)
+        x = Conv2D(nb_filter, (3, 3), kernel_initializer='he_normal', padding='same', use_bias=False,
+                   name=name_or_none(block_prefix, '_conv2D'))(x)
         if dropout_rate:
             x = Dropout(dropout_rate)(x)
 
@@ -557,7 +564,7 @@ def __conv_block(ip, nb_filter, bottleneck=False, dropout_rate=None, weight_deca
 
 
 def __dense_block(x, nb_layers, nb_filter, growth_rate, bottleneck=False, dropout_rate=None,
-                  weight_decay=1e-4, grow_nb_filters=True, return_concat_list=False):
+                  weight_decay=1e-4, grow_nb_filters=True, return_concat_list=False, block_prefix=None):
     '''
     Build a dense_block where the output of each conv_block is fed
     to subsequent ones
@@ -575,6 +582,7 @@ def __dense_block(x, nb_layers, nb_filter, growth_rate, bottleneck=False, dropou
         grow_nb_filters: if True, allows number of filters to grow
         return_concat_list: set to True to return the list of
             feature maps along with the actual output
+        block_prefix: str, for block unique naming
 
     # Return
         If return_concat_list is True, returns a list of the output
@@ -590,7 +598,8 @@ def __dense_block(x, nb_layers, nb_filter, growth_rate, bottleneck=False, dropou
         x_list = [x]
 
         for i in range(nb_layers):
-            cb = __conv_block(x, growth_rate, bottleneck, dropout_rate, weight_decay)
+            cb = __conv_block(x, growth_rate, bottleneck, dropout_rate, weight_decay,
+                              block_prefix=name_or_none(block_prefix, '_%i' % i))
             x_list.append(cb)
 
             x = concatenate([x, cb], axis=concat_axis)
@@ -604,7 +613,7 @@ def __dense_block(x, nb_layers, nb_filter, growth_rate, bottleneck=False, dropou
             return x, nb_filter
 
 
-def __transition_block(ip, nb_filter, compression=1.0, weight_decay=1e-4):
+def __transition_block(ip, nb_filter, compression=1.0, weight_decay=1e-4, block_prefix=None):
     '''
     Adds a pointwise convolution layer (with batch normalization and relu),
     and an average pooling layer. The number of output convolution filters
@@ -617,6 +626,7 @@ def __transition_block(ip, nb_filter, compression=1.0, weight_decay=1e-4):
         compression: calculated as 1 - reduction. Reduces the number
             of feature maps in the transition block.
         weight_decay: weight decay factor
+        block_prefix: str, for block unique naming
 
     # Input shape
         4D tensor with shape:
@@ -638,16 +648,16 @@ def __transition_block(ip, nb_filter, compression=1.0, weight_decay=1e-4):
     with K.name_scope('Transition'):
         concat_axis = 1 if K.image_data_format() == 'channels_first' else -1
 
-        x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(ip)
+        x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5, name=name_or_none(block_prefix, '_bn'))(ip)
         x = Activation('relu')(x)
         x = Conv2D(int(nb_filter * compression), (1, 1), kernel_initializer='he_normal', padding='same',
-                   use_bias=False, kernel_regularizer=l2(weight_decay))(x)
+                   use_bias=False, kernel_regularizer=l2(weight_decay), name=name_or_none(block_prefix, '_conv2D'))(x)
         x = AveragePooling2D((2, 2), strides=(2, 2))(x)
 
         return x
 
 
-def __transition_up_block(ip, nb_filters, type='deconv', weight_decay=1E-4):
+def __transition_up_block(ip, nb_filters, type='deconv', weight_decay=1E-4, block_prefix=None):
     '''Adds an upsampling block. Upsampling operation relies on the the type parameter.
 
     # Arguments
@@ -657,6 +667,7 @@ def __transition_up_block(ip, nb_filters, type='deconv', weight_decay=1E-4):
         type: can be 'upsampling', 'subpixel', 'deconv'. Determines
             type of upsampling performed
         weight_decay: weight decay factor
+        block_prefix: str, for block unique naming
 
     # Input shape
         4D tensor with shape:
@@ -676,17 +687,17 @@ def __transition_up_block(ip, nb_filters, type='deconv', weight_decay=1E-4):
     with K.name_scope('TransitionUp'):
 
         if type == 'upsampling':
-            x = UpSampling2D()(ip)
+            x = UpSampling2D(name=name_or_none(block_prefix, '_upsampling'))(ip)
         elif type == 'subpixel':
             x = Conv2D(nb_filters, (3, 3), activation='relu', padding='same', kernel_regularizer=l2(weight_decay),
-                       use_bias=False, kernel_initializer='he_normal')(ip)
-            x = SubPixelUpscaling(scale_factor=2)(x)
+                       use_bias=False, kernel_initializer='he_normal', name=name_or_none(block_prefix, '_conv2D'))(ip)
+            x = SubPixelUpscaling(scale_factor=2, name=name_or_none(block_prefix, '_subpixel'))(x)
             x = Conv2D(nb_filters, (3, 3), activation='relu', padding='same', kernel_regularizer=l2(weight_decay),
-                       use_bias=False, kernel_initializer='he_normal')(x)
+                       use_bias=False, kernel_initializer='he_normal', name=name_or_none(block_prefix, '_conv2D'))(x)
         else:
             x = Conv2DTranspose(nb_filters, (3, 3), activation='relu', padding='same', strides=(2, 2),
-                                kernel_initializer='he_normal', kernel_regularizer=l2(weight_decay))(ip)
-
+                                kernel_initializer='he_normal', kernel_regularizer=l2(weight_decay),
+                                name=name_or_none(block_prefix, '_conv2DT'))(ip)
         return x
 
 
@@ -781,27 +792,30 @@ def __create_dense_net(nb_classes, img_input, include_top, depth=40, nb_dense_bl
             initial_kernel = (3, 3)
             initial_strides = (1, 1)
 
-        x = Conv2D(nb_filter, initial_kernel, kernel_initializer='he_normal', padding='same',
+        x = Conv2D(nb_filter, initial_kernel, kernel_initializer='he_normal', padding='same', name='initial_conv2D',
                    strides=initial_strides, use_bias=False, kernel_regularizer=l2(weight_decay))(img_input)
 
         if subsample_initial_block:
-            x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(x)
+            x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5, name='initial_bn')(x)
             x = Activation('relu')(x)
             x = MaxPooling2D((3, 3), strides=(2, 2), padding='same')(x)
 
         # Add dense blocks
         for block_idx in range(nb_dense_block - 1):
             x, nb_filter = __dense_block(x, nb_layers[block_idx], nb_filter, growth_rate, bottleneck=bottleneck,
-                                         dropout_rate=dropout_rate, weight_decay=weight_decay)
+                                         dropout_rate=dropout_rate, weight_decay=weight_decay,
+                                         block_prefix='dense_%i' % block_idx)
             # add transition_block
-            x = __transition_block(x, nb_filter, compression=compression, weight_decay=weight_decay)
+            x = __transition_block(x, nb_filter, compression=compression, weight_decay=weight_decay,
+                                   block_prefix='tr_%i' % block_idx)
             nb_filter = int(nb_filter * compression)
 
         # The last dense_block does not have a transition_block
         x, nb_filter = __dense_block(x, final_nb_layer, nb_filter, growth_rate, bottleneck=bottleneck,
-                                     dropout_rate=dropout_rate, weight_decay=weight_decay)
+                                     dropout_rate=dropout_rate, weight_decay=weight_decay,
+                                     block_prefix='dense_%i' % (nb_dense_block - 1))
 
-        x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(x)
+        x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5, name='final_bn')(x)
         x = Activation('relu')(x)
 
         if include_top:
@@ -889,7 +903,7 @@ def __create_fcn_dense_net(nb_classes, img_input, include_top, nb_dense_block=5,
         # Initial convolution
         x = Conv2D(init_conv_filters, (7, 7), kernel_initializer='he_normal', padding='same', name='initial_conv2D',
                    use_bias=False, kernel_regularizer=l2(weight_decay))(img_input)
-        x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(x)
+        x = BatchNormalization(axis=concat_axis, epsilon=1.1e-5, name='initial_bn')(x)
         x = Activation('relu')(x)
 
         nb_filter = init_conv_filters
@@ -899,13 +913,14 @@ def __create_fcn_dense_net(nb_classes, img_input, include_top, nb_dense_block=5,
         # Add dense blocks and transition down block
         for block_idx in range(nb_dense_block):
             x, nb_filter = __dense_block(x, nb_layers[block_idx], nb_filter, growth_rate, dropout_rate=dropout_rate,
-                                         weight_decay=weight_decay)
+                                         weight_decay=weight_decay, block_prefix='dense_%i' % block_idx)
 
             # Skip connection
             skip_list.append(x)
 
             # add transition_block
-            x = __transition_block(x, nb_filter, compression=compression, weight_decay=weight_decay)
+            x = __transition_block(x, nb_filter, compression=compression, weight_decay=weight_decay,
+                                   block_prefix='tr_%i' % block_idx)
 
             nb_filter = int(nb_filter * compression)  # this is calculated inside transition_down_block
 
@@ -913,7 +928,8 @@ def __create_fcn_dense_net(nb_classes, img_input, include_top, nb_dense_block=5,
         # return the concatenated feature maps without the concatenation of the input
         _, nb_filter, concat_list = __dense_block(x, bottleneck_nb_layers, nb_filter, growth_rate,
                                                   dropout_rate=dropout_rate, weight_decay=weight_decay,
-                                                  return_concat_list=True)
+                                                  return_concat_list=True,
+                                                  block_prefix='dense_%i' % nb_dense_block)
 
         skip_list = skip_list[::-1]  # reverse the skip list
 
@@ -925,16 +941,18 @@ def __create_fcn_dense_net(nb_classes, img_input, include_top, nb_dense_block=5,
             # not the concatenation of the input with the feature maps (concat_list[0].
             l = concatenate(concat_list[1:], axis=concat_axis)
 
-            t = __transition_up_block(l, nb_filters=n_filters_keep, type=upsampling_type, weight_decay=weight_decay)
+            t = __transition_up_block(l, nb_filters=n_filters_keep, type=upsampling_type, weight_decay=weight_decay,
+                                      block_prefix='tr_up_%i' % block_idx)
 
             # concatenate the skip connection with the transition block
             x = concatenate([t, skip_list[block_idx]], axis=concat_axis)
 
             # Dont allow the feature map size to grow in upsampling dense blocks
-            x_up, nb_filter, concat_list = __dense_block(x, nb_layers[nb_dense_block + block_idx + 1], nb_filter=growth_rate,
-                                                         growth_rate=growth_rate, dropout_rate=dropout_rate,
-                                                         weight_decay=weight_decay, return_concat_list=True,
-                                                         grow_nb_filters=False)
+            x_up, nb_filter, concat_list = __dense_block(x, nb_layers[nb_dense_block + block_idx + 1],
+                                                         nb_filter=growth_rate, growth_rate=growth_rate,
+                                                         dropout_rate=dropout_rate, weight_decay=weight_decay,
+                                                         return_concat_list=True, grow_nb_filters=False,
+                                                         block_prefix='dense_%i' % (nb_dense_block + 1 + block_idx))
 
         if include_top:
             x = Conv2D(nb_classes, (1, 1), activation='linear', padding='same', use_bias=False)(x_up)

keras_contrib/applications/nasnet.py

@@ -0,0 +1,653 @@
+"""Collection of NASNet models
+
+The reference paper:
+ - [Learning Transferable Architectures for Scalable Image Recognition]
+    (https://arxiv.org/abs/1707.07012)
+
+The reference implementation:
+1. TF Slim
+ - https://github.com/tensorflow/models/blob/master/research/slim/nets/
+   nasnet/nasnet.py
+2. TensorNets
+ - https://github.com/taehoonlee/tensornets/blob/master/tensornets/nasnets.py
+"""
+from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import division
+
+import warnings
+
+from keras.models import Model
+from keras.layers import Input
+from keras.layers import Activation
+from keras.layers import Dense
+from keras.layers import Dropout
+from keras.layers import BatchNormalization
+from keras.layers import MaxPooling2D
+from keras.layers import AveragePooling2D
+from keras.layers import GlobalAveragePooling2D
+from keras.layers import GlobalMaxPooling2D
+from keras.layers import Conv2D
+from keras.layers import SeparableConv2D
+from keras.layers import ZeroPadding2D
+from keras.layers import Cropping2D
+from keras.layers import concatenate
+from keras.layers import add
+from keras.utils.data_utils import get_file
+from keras.engine.topology import get_source_inputs
+from keras.applications.imagenet_utils import _obtain_input_shape
+from keras.applications.inception_v3 import preprocess_input
+from keras.applications.imagenet_utils import decode_predictions
+from keras import backend as K
+
+_BN_DECAY = 0.9997
+_BN_EPSILON = 1e-3
+
+
+def NASNet(input_shape=None,
+           penultimate_filters=4032,
+           nb_blocks=6,
+           stem_filters=96,
+           skip_reduction=True,
+           use_auxilary_branch=False,
+           filters_multiplier=2,
+           dropout=0.5,
+           include_top=True,
+           weights=None,
+           input_tensor=None,
+           pooling=None,
+           classes=1000,
+           default_size=None):
+    """Instantiates a NASNet architecture.
+    Note that only TensorFlow is supported for now,
+    therefore it only works with the data format
+    `image_data_format='channels_last'` in your Keras config
+    at `~/.keras/keras.json`.
+
+    # Arguments
+        input_shape: optional shape tuple, only to be specified
+            if `include_top` is False (otherwise the input shape
+            has to be `(331, 331, 3)` for NASNetLarge or
+            `(224, 224, 3)` for NASNetMobile
+            It should have exactly 3 inputs channels,
+            and width and height should be no smaller than 32.
+            E.g. `(224, 224, 3)` would be one valid value.
+        penultimate_filters: number of filters in the penultimate layer.
+            NASNet models use the notation `NASNet (N @ P)`, where:
+                -   N is the number of blocks
+                -   P is the number of penultimate filters
+        nb_blocks: number of repeated blocks of the NASNet model.
+            NASNet models use the notation `NASNet (N @ P)`, where:
+                -   N is the number of blocks
+                -   P is the number of penultimate filters
+        stem_filters: number of filters in the initial stem block
+        skip_reduction: Whether to skip the reduction step at the tail
+            end of the network. Set to `False` for CIFAR models.
+        use_auxilary_branch: Whether to use the auxilary branch during
+            training or evaluation.
+        filters_multiplier: controls the width of the network.
+            - If `filters_multiplier` < 1.0, proportionally decreases the number
+                of filters in each layer.
+            - If `filters_multiplier` > 1.0, proportionally increases the number
+                of filters in each layer.
+            - If `filters_multiplier` = 1, default number of filters from the paper
+                 are used at each layer.
+        dropout: dropout rate
+        include_top: whether to include the fully-connected
+            layer at the top of the network.
+        weights: `None` (random initialization) or
+            `imagenet` (ImageNet weights)
+        input_tensor: optional Keras tensor (i.e. output of
+            `layers.Input()`)
+            to use as image input for the model.
+        pooling: Optional pooling mode for feature extraction
+            when `include_top` is `False`.
+            - `None` means that the output of the model
+                will be the 4D tensor output of the
+                last convolutional layer.
+            - `avg` means that global average pooling
+                will be applied to the output of the
+                last convolutional layer, and thus
+                the output of the model will be a
+                2D tensor.
+            - `max` means that global max pooling will
+                be applied.
+        classes: optional number of classes to classify images
+            into, only to be specified if `include_top` is True, and
+            if no `weights` argument is specified.
+        default_size: specifies the default image size of the model
+    # Returns
+        A Keras model instance.
+    # Raises
+        ValueError: in case of invalid argument for `weights`,
+            or invalid input shape.
+        RuntimeError: If attempting to run this model with a
+            backend that does not support separable convolutions.
+    """
+    if K.backend() != 'tensorflow':
+        raise RuntimeError('Only Tensorflow backend is currently supported, '
+                           'as other backends do not support '
+                           'separable convolution.')
+
+    if weights not in {'imagenet', None}:
+        raise ValueError('The `weights` argument should be either '
+                         '`None` (random initialization) or `imagenet` '
+                         '(pre-training on ImageNet).')
+
+    if weights == 'imagenet' and include_top and classes != 1000:
+        raise ValueError('If using `weights` as ImageNet with `include_top` '
+                         'as true, `classes` should be 1000')
+
+    if default_size is None:
+        default_size = 331
+
+    # Determine proper input shape and default size.
+    input_shape = _obtain_input_shape(input_shape,
+                                      default_size=default_size,
+                                      min_size=32,
+                                      data_format=K.image_data_format(),
+                                      require_flatten=include_top or weights)
+
+    if K.image_data_format() != 'channels_last':
+        warnings.warn('The MobileNet family of models is only available '
+                      'for the input data format "channels_last" '
+                      '(width, height, channels). '
+                      'However your settings specify the default '
+                      'data format "channels_first" (channels, width, height).'
+                      ' You should set `image_data_format="channels_last"` '
+                      'in your Keras config located at ~/.keras/keras.json. '
+                      'The model being returned right now will expect inputs '
+                      'to follow the "channels_last" data format.')
+        K.set_image_data_format('channels_last')
+        old_data_format = 'channels_first'
+    else:
+        old_data_format = None
+
+    if input_tensor is None:
+        img_input = Input(shape=input_shape)
+    else:
+        if not K.is_keras_tensor(input_tensor):
+            img_input = Input(tensor=input_tensor, shape=input_shape)
+        else:
+            img_input = input_tensor
+
+    assert penultimate_filters % 24 == 0, "`penultimate_filters` needs to be divisible " \
+                                          "by 6 * (2^N)."
+
+    channel_dim = 1 if K.image_data_format() == 'channels_first' else -1
+    filters = penultimate_filters // 24
+
+    x = Conv2D(stem_filters, (3, 3), strides=(2, 2), padding='valid', use_bias=False, name='stem_conv1',
+               kernel_initializer='he_normal')(img_input)
+    x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON,
+                           name='stem_bn1')(x)
+
+    x, p = _reduction_A(x, None, filters // (filters_multiplier ** 2), id='stem_1')
+    x, p = _reduction_A(x, p, filters // filters_multiplier, id='stem_2')
+
+    for i in range(nb_blocks):
+        x, p = _normal_A(x, p, filters, id='%d' % (i))
+
+    x, p0 = _reduction_A(x, p, filters * filters_multiplier, id='reduce_%d' % (nb_blocks))
+
+    p = p0 if not skip_reduction else p
+
+    for i in range(nb_blocks):
+        x, p = _normal_A(x, p, filters * filters_multiplier, id='%d' % (nb_blocks + i + 1))
+
+    auxilary_x = None
+    if use_auxilary_branch:
+        img_height = 1 if K.image_data_format() == 'channels_first' else 2
+        img_width = 2 if K.image_data_format() == 'channels_first' else 3
+
+        with K.name_scope('auxilary_branch'):
+            auxilary_x = Activation('relu')(x)
+            auxilary_x = AveragePooling2D((5, 5), strides=(3, 3), padding='valid', name='aux_pool')(auxilary_x)
+            auxilary_x = Conv2D(128, (1, 1), padding='same', use_bias=False, name='aux_conv_projection',
+                                kernel_initializer='he_normal')(auxilary_x)
+            auxilary_x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON,
+                                            name='aux_bn_projection')(auxilary_x)
+            auxilary_x = Activation('relu')(auxilary_x)
+
+            auxilary_x = Conv2D(768, (auxilary_x._keras_shape[img_height], auxilary_x._keras_shape[img_width]),
+                                padding='valid', use_bias=False, kernel_initializer='he_normal',
+                                name='aux_conv_reduction')(auxilary_x)
+            auxilary_x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON,
+                                            name='aux_bn_reduction')(auxilary_x)
+            auxilary_x = Activation('relu')(auxilary_x)
+
+            auxilary_x = GlobalAveragePooling2D()(auxilary_x)
+            auxilary_x = Dense(classes, activation='softmax', name='aux_predictions')(auxilary_x)
+
+    x, p0 = _reduction_A(x, p, filters * filters_multiplier ** 2, id='reduce_%d' % (2 * nb_blocks))
+
+    p = p0 if not skip_reduction else p
+
+    for i in range(nb_blocks):
+        x, p = _normal_A(x, p, filters * filters_multiplier ** 2, id='%d' % (2 * nb_blocks + i + 1))
+
+    x = Activation('relu')(x)
+
+    if include_top:
+        x = GlobalAveragePooling2D()(x)
+        x = Dropout(dropout)(x)
+        x = Dense(classes, activation='softmax')(x)
+    else:
+        if pooling == 'avg':
+            x = GlobalAveragePooling2D()(x)
+        elif pooling == 'max':
+            x = GlobalMaxPooling2D()(x)
+
+    # Ensure that the model takes into account
+    # any potential predecessors of `input_tensor`.
+    if input_tensor is not None:
+        inputs = get_source_inputs(input_tensor)
+    else:
+        inputs = img_input
+
+    # Create model.
+    if use_auxilary_branch:
+        model = Model(inputs, [x, auxilary_x], name='NASNet_with_auxilary')
+    else:
+        model = Model(inputs, x, name='NASNet')
+
+    # load weights (when available)
+    warnings.warn('Weights of NASNet models have not been ported yet for Keras.')
+
+    if old_data_format:
+        K.set_image_data_format(old_data_format)
+
+    return model
+
+
+def NASNetLarge(input_shape=None,
+                dropout=0.5,
+                use_auxilary_branch=False,
+                include_top=True,
+                weights='imagenet',
+                input_tensor=None,
+                pooling=None,
+                classes=1000):
+    """Instantiates a NASNet architecture in ImageNet mode.
+    Note that only TensorFlow is supported for now,
+    therefore it only works with the data format
+    `image_data_format='channels_last'` in your Keras config
+    at `~/.keras/keras.json`.
+
+    # Arguments
+        input_shape: optional shape tuple, only to be specified
+            if `include_top` is False (otherwise the input shape
+            has to be `(331, 331, 3)` for NASNetLarge.
+            It should have exactly 3 inputs channels,
+            and width and height should be no smaller than 32.
+            E.g. `(224, 224, 3)` would be one valid value.
+        use_auxilary_branch: Whether to use the auxilary branch during
+            training or evaluation.
+        dropout: dropout rate
+        include_top: whether to include the fully-connected
+            layer at the top of the network.
+        weights: `None` (random initialization) or
+            `imagenet` (ImageNet weights)
+        input_tensor: optional Keras tensor (i.e. output of
+            `layers.Input()`)
+            to use as image input for the model.
+        pooling: Optional pooling mode for feature extraction
+            when `include_top` is `False`.
+            - `None` means that the output of the model
+                will be the 4D tensor output of the
+                last convolutional layer.
+            - `avg` means that global average pooling
+                will be applied to the output of the
+                last convolutional layer, and thus
+                the output of the model will be a
+                2D tensor.
+            - `max` means that global max pooling will
+                be applied.
+        classes: optional number of classes to classify images
+            into, only to be specified if `include_top` is True, and
+            if no `weights` argument is specified.
+        default_size: specifies the default image size of the model
+    # Returns
+        A Keras model instance.
+    # Raises
+        ValueError: in case of invalid argument for `weights`,
+            or invalid input shape.
+        RuntimeError: If attempting to run this model with a
+            backend that does not support separable convolutions.
+    """
+    return NASNet(input_shape,
+                  penultimate_filters=4032,
+                  nb_blocks=6,
+                  stem_filters=96,
+                  skip_reduction=True,
+                  use_auxilary_branch=use_auxilary_branch,
+                  filters_multiplier=2,
+                  dropout=dropout,
+                  include_top=include_top,
+                  weights=weights,
+                  input_tensor=input_tensor,
+                  pooling=pooling,
+                  classes=classes,
+                  default_size=331)
+
+
+def NASNetMobile(input_shape=None,
+                 dropout=0.5,
+                 use_auxilary_branch=False,
+                 include_top=True,
+                 weights='imagenet',
+                 input_tensor=None,
+                 pooling=None,
+                 classes=1000):
+    """Instantiates a NASNet architecture in Mobile ImageNet mode.
+    Note that only TensorFlow is supported for now,
+    therefore it only works with the data format
+    `image_data_format='channels_last'` in your Keras config
+    at `~/.keras/keras.json`.
+
+    # Arguments
+        input_shape: optional shape tuple, only to be specified
+            if `include_top` is False (otherwise the input shape
+            has to be `(224, 224, 3)` for NASNetMobile
+            It should have exactly 3 inputs channels,
+            and width and height should be no smaller than 32.
+            E.g. `(224, 224, 3)` would be one valid value.
+        use_auxilary_branch: Whether to use the auxilary branch during
+            training or evaluation.
+        dropout: dropout rate
+        include_top: whether to include the fully-connected
+            layer at the top of the network.
+        weights: `None` (random initialization) or
+            `imagenet` (ImageNet weights)
+        input_tensor: optional Keras tensor (i.e. output of
+            `layers.Input()`)
+            to use as image input for the model.
+        pooling: Optional pooling mode for feature extraction
+            when `include_top` is `False`.
+            - `None` means that the output of the model
+                will be the 4D tensor output of the
+                last convolutional layer.
+            - `avg` means that global average pooling
+                will be applied to the output of the
+                last convolutional layer, and thus
+                the output of the model will be a
+                2D tensor.
+            - `max` means that global max pooling will
+                be applied.
+        classes: optional number of classes to classify images
+            into, only to be specified if `include_top` is True, and
+            if no `weights` argument is specified.
+        default_size: specifies the default image size of the model
+    # Returns
+        A Keras model instance.
+    # Raises
+        ValueError: in case of invalid argument for `weights`,
+            or invalid input shape.
+        RuntimeError: If attempting to run this model with a
+            backend that does not support separable convolutions.
+    """
+    return NASNet(input_shape,
+                  penultimate_filters=1056,
+                  nb_blocks=4,
+                  stem_filters=32,
+                  skip_reduction=False,
+                  use_auxilary_branch=use_auxilary_branch,
+                  filters_multiplier=2,
+                  dropout=dropout,
+                  include_top=include_top,
+                  weights=weights,
+                  input_tensor=input_tensor,
+                  pooling=pooling,
+                  classes=classes,
+                  default_size=224)
+
+
+def NASNetCIFAR(input_shape=None,
+                dropout=0.0,
+                use_auxilary_branch=False,
+                include_top=True,
+                weights=None,
+                input_tensor=None,
+                pooling=None,
+                classes=10):
+    """Instantiates a NASNet architecture in CIFAR mode.
+    Note that only TensorFlow is supported for now,
+    therefore it only works with the data format
+    `image_data_format='channels_last'` in your Keras config
+    at `~/.keras/keras.json`.
+
+    # Arguments
+        input_shape: optional shape tuple, only to be specified
+            if `include_top` is False (otherwise the input shape
+            has to be `(32, 32, 3)` for NASNetMobile
+            It should have exactly 3 inputs channels,
+            and width and height should be no smaller than 32.
+            E.g. `(32, 32, 3)` would be one valid value.
+        use_auxilary_branch: Whether to use the auxilary branch during
+            training or evaluation.
+        dropout: dropout rate
+        include_top: whether to include the fully-connected
+            layer at the top of the network.
+        weights: `None` (random initialization) or
+            `imagenet` (ImageNet weights)
+        input_tensor: optional Keras tensor (i.e. output of
+            `layers.Input()`)
+            to use as image input for the model.
+        pooling: Optional pooling mode for feature extraction
+            when `include_top` is `False`.
+            - `None` means that the output of the model
+                will be the 4D tensor output of the
+                last convolutional layer.
+            - `avg` means that global average pooling
+                will be applied to the output of the
+                last convolutional layer, and thus
+                the output of the model will be a
+                2D tensor.
+            - `max` means that global max pooling will
+                be applied.
+        classes: optional number of classes to classify images
+            into, only to be specified if `include_top` is True, and
+            if no `weights` argument is specified.
+        default_size: specifies the default image size of the model
+    # Returns
+        A Keras model instance.
+    # Raises
+        ValueError: in case of invalid argument for `weights`,
+            or invalid input shape.
+        RuntimeError: If attempting to run this model with a
+            backend that does not support separable convolutions.
+    """
+    return NASNet(input_shape,
+                  penultimate_filters=768,
+                  nb_blocks=6,
+                  stem_filters=96,
+                  skip_reduction=True,
+                  use_auxilary_branch=use_auxilary_branch,
+                  filters_multiplier=2,
+                  dropout=dropout,
+                  include_top=include_top,
+                  weights=weights,
+                  input_tensor=input_tensor,
+                  pooling=pooling,
+                  classes=classes,
+                  default_size=224)
+
+
+def _separable_conv_block(ip, filters, kernel_size=(3, 3), strides=(1, 1), id=None):
+    '''Adds 2 blocks of [relu-separable conv-batchnorm]
+
+    # Arguments:
+        ip: input tensor
+        filters: number of output filters per layer
+        kernel_size: kernel size of separable convolutions
+        strides: strided convolution for downsampling
+        id: string id
+
+    # Returns:
+        a Keras tensor
+    '''
+    channel_dim = 1 if K.image_data_format() == 'channels_first' else -1
+
+    with K.name_scope('separable_conv_block_%s' % id):
+        x = Activation('relu')(ip)
+        x = SeparableConv2D(filters, kernel_size, strides=strides, name='separable_conv_1_%s' % id,
+                            padding='same', use_bias=False, kernel_initializer='he_normal')(x)
+        x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON,
+                               name="separable_conv_1_bn_%s" % (id))(x)
+        x = Activation('relu')(x)
+        x = SeparableConv2D(filters, kernel_size, name='separable_conv_2_%s' % id,
+                            padding='same', use_bias=False, kernel_initializer='he_normal')(x)
+        x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON,
+                               name="separable_conv_2_bn_%s" % (id))(x)
+    return x
+
+
+def _adjust_block(p, ip, filters, id=None):
+    '''
+    Adjusts the input `p` to match the shape of the `input`
+    or situations where the output number of filters needs to
+    be changed
+
+    # Arguments:
+        p: input tensor which needs to be modified
+        ip: input tensor whose shape needs to be matched
+        filters: number of output filters to be matched
+        id: string id
+
+    # Returns:
+        an adjusted Keras tensor
+    '''
+    channel_dim = 1 if K.image_data_format() == 'channels_first' else -1
+    img_dim = 2 if K.image_data_format() == 'channels_first' else -2
+
+    with K.name_scope('adjust_block'):
+        if p is None:
+            p = ip
+
+        elif p._keras_shape[img_dim] != ip._keras_shape[img_dim]:
+            with K.name_scope('adjust_reduction_block_%s' % id):
+                p = Activation('relu', name='adjust_relu_1_%s' % id)(p)
+
+                p1 = AveragePooling2D((1, 1), strides=(2, 2), padding='valid', name='adjust_avg_pool_1_%s' % id)(p)
+                p1 = Conv2D(filters // 2, (1, 1), padding='same', use_bias=False,
+                            name='adjust_conv_1_%s' % id, kernel_initializer='he_normal')(p1)
+
+                p2 = ZeroPadding2D(padding=((0, 1), (0, 1)))(p)
+                p2 = Cropping2D(cropping=((1, 0), (1, 0)))(p2)
+                p2 = AveragePooling2D((1, 1), strides=(2, 2), padding='valid', name='adjust_avg_pool_2_%s' % id)(p2)
+                p2 = Conv2D(filters // 2, (1, 1), padding='same', use_bias=False,
+                            name='adjust_conv_2_%s' % id, kernel_initializer='he_normal')(p2)
+
+                p = concatenate([p1, p2], axis=channel_dim)
+                p = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON,
+                                       name='adjust_bn_%s' % id)(p)
+
+        elif p._keras_shape[channel_dim] != filters:
+            with K.name_scope('adjust_projection_block_%s' % id):
+                p = Activation('relu')(p)
+                p = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='adjust_conv_projection_%s' % id,
+                           use_bias=False, kernel_initializer='he_normal')(p)
+                p = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON,
+                                       name='adjust_bn_%s' % id)(p)
+    return p
+
+
+def _normal_A(ip, p, filters, id=None):
+    '''Adds a Normal cell for NASNet-A (Fig. 4 in the paper)
+
+    # Arguments:
+        ip: input tensor `x`
+        p: input tensor `p`
+        filters: number of output filters
+        id: string id
+
+    # Returns:
+        a Keras tensor
+    '''
+    channel_dim = 1 if K.image_data_format() == 'channels_first' else -1
+
+    with K.name_scope('normal_A_block_%s' % id):
+        p = _adjust_block(p, ip, filters, id)
+
+        h = Activation('relu')(ip)
+        h = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='normal_conv_1_%s' % id,
+                   use_bias=False, kernel_initializer='he_normal')(h)
+        h = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON,
+                               name='normal_bn_1_%s' % id)(h)
+
+        with K.name_scope('block_1'):
+            x1 = _separable_conv_block(h, filters, id='normal_left1_%s' % id)
+            x1 = add([x1, h], name='normal_add_1_%s' % id)
+
+        with K.name_scope('block_2'):
+            x2_1 = _separable_conv_block(p, filters, id='normal_left2_%s' % id)
+            x2_2 = _separable_conv_block(h, filters, kernel_size=(5, 5), id='normal_right2_%s' % id)
+            x2 = add([x2_1, x2_2], name='normal_add_2_%s' % id)
+
+        with K.name_scope('block_3'):
+            x3 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_left3_%s' % (id))(h)
+            x3 = add([x3, p], name='normal_add_3_%s' % id)
+
+        with K.name_scope('block_4'):
+            x4_1 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_left4_%s' % (id))(p)
+            x4_2 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='normal_right4_%s' % (id))(p)
+            x4 = add([x4_1, x4_2], name='normal_add_4_%s' % id)
+
+        with K.name_scope('block_5'):
+            x5_1 = _separable_conv_block(p, filters, (5, 5), id='normal_left5_%s' % id)
+            x5_2 = _separable_conv_block(p, filters, (3, 3), id='normal_right5_%s' % id)
+            x5 = add([x5_1, x5_2], name='normal_add_5_%s' % id)
+
+        x = concatenate([p, x2, x5, x3, x4, x1], axis=channel_dim, name='normal_concat_%s' % id)
+    return x, ip
+
+
+def _reduction_A(ip, p, filters, id=None):
+    '''Adds a Reduction cell for NASNet-A (Fig. 4 in the paper)
+
+    # Arguments:
+        ip: input tensor `x`
+        p: input tensor `p`
+        filters: number of output filters
+        id: string id
+
+    # Returns:
+        a Keras tensor
+    '''
+    """"""
+    channel_dim = 1 if K.image_data_format() == 'channels_first' else -1
+
+    with K.name_scope('reduction_A_block_%s' % id):
+        p = _adjust_block(p, ip, filters, id)
+
+        h = Activation('relu')(ip)
+        h = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='reduction_conv_1_%s' % id,
+                   use_bias=False, kernel_initializer='he_normal')(h)
+        h = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON,
+                               name='reduction_bn_1_%s' % id)(h)
+
+        with K.name_scope('block_1'):
+            x1_1 = _separable_conv_block(p, filters, (7, 7), strides=(2, 2), id='reduction_left1_%s' % id)
+            x1_2 = _separable_conv_block(h, filters, (5, 5), strides=(2, 2), id='reduction_right1_%s' % id)
+            x1 = add([x1_1, x1_2], name='reduction_add_1_%s' % id)
+
+        with K.name_scope('block_2'):
+            x2_1 = MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left2_%s' % id)(h)
+            x2_2 = _separable_conv_block(p, filters, (7, 7), strides=(2, 2), id='reduction_right2_%s' % id)
+            x2 = add([x2_1, x2_2], name='reduction_add_2_%s' % id)
+
+        with K.name_scope('block_3'):
+            x3_1 = AveragePooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left3_%s' % id)(h)
+            x3_2 = _separable_conv_block(p, filters, (5, 5), strides=(2, 2), id='reduction_right3_%s' % id)
+            x3 = add([x3_1, x3_2], name='reduction_add3_%s' % id)
+
+        with K.name_scope('block_4'):
+            x4_1 = MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left4_%s' % id)(h)
+            x4_2 = _separable_conv_block(x1, filters, (3, 3), id='reduction_right4_%s' % id)
+            x4 = add([x4_1, x4_2], name='reduction_add4_%s' % id)
+
+        with K.name_scope('block_5'):
+            x5 = AveragePooling2D((3, 3), strides=(1, 1), padding='same', name='reduction_left5_%s' % id)(x1)
+
+        x = concatenate([x2, x3, x5, x4], axis=channel_dim, name='reduction_concat_%s' % id)
+        return x, ip

keras_contrib/callbacks/dead_relu_detector.py

@@ -1,8 +1,6 @@
 import numpy as np
-import warnings
 
 from keras.callbacks import Callback
-from keras.layers import Dense
 from keras import backend as K
 
 
@@ -13,10 +11,11 @@ class DeadReluDetector(Callback):
     # Arguments
         x_train: Training dataset to check whether or not neurons fire
         verbose: verbosity mode
-            True means that even a single dead neuron triggers warning
+            True means that even a single dead neuron triggers a warning message
             False means that only significant number of dead neurons (10% or more)
-            triggers warning
+            triggers a warning message
     """
+
     def __init__(self, x_train, verbose=False):
         super(DeadReluDetector, self).__init__()
         self.x_train = x_train
@@ -25,7 +24,8 @@ class DeadReluDetector(Callback):
 
     @staticmethod
     def is_relu_layer(layer):
-        return isinstance(layer, Dense) and layer.get_config()['activation'] == 'relu'
+        # Should work for all layers with relu activation. Tested for Dense and Conv2D
+        return 'activation' in layer.get_config() and layer.get_config()['activation'] == 'relu'
 
     def get_relu_activations(self):
         model_input = self.model.input
@@ -44,17 +44,43 @@ class DeadReluDetector(Callback):
         layer_outputs = [func(list_inputs)[0] for func in funcs]
         for layer_index, layer_activations in enumerate(layer_outputs):
             if self.is_relu_layer(self.model.layers[layer_index]):
-                yield [layer_index, layer_activations]
+                layer_name = self.model.layers[layer_index].name
+                # layer_weight is a list [W] (+ [b])
+                layer_weight = self.model.layers[layer_index].get_weights()
+                # with kernel and bias, the weights are saved as a list [W, b]. If only weights, it is [W]
+                if type(layer_weight) is not list:
+                    raise ValueError("'Layer_weight' should be a list, but was {}".format(type(layer_weight)))
+
+                layer_weight_shape = np.shape(layer_weight[0])
+                yield [layer_index, layer_activations, layer_name, layer_weight_shape]
 
     def on_epoch_end(self, epoch, logs={}):
         for relu_activation in self.get_relu_activations():
-            layer_index, activation_values = relu_activation
-            total_neurons = activation_values.shape[-1]
-            dead_neurons = np.sum(activation_values == 0)
-            dead_neurons_share = dead_neurons / total_neurons
-            if (self.verbose and dead_neurons > 0) or dead_neurons_share > self.dead_neurons_share_threshold:
-                warnings.warn(
-                    'Layer #{} has {} dead neurons ({:.2%})!'
-                        .format(layer_index, dead_neurons, dead_neurons_share),
-                    RuntimeWarning
-                )
+            layer_index, activation_values, layer_name, layer_weight_shape = relu_activation
+
+            shape_act = activation_values.shape
+
+            weight_len = len(layer_weight_shape)
+            act_len = len(shape_act)
+
+            # should work for both Conv and Flat
+            if K.image_data_format() == 'channels_last':
+                # features in last axis
+                axis_filter = -1
+            else:
+                # features before the convolution axis, for weight_len the input and output have to be subtracted
+                axis_filter = -1 - (weight_len - 2)
+
+            total_featuremaps = shape_act[axis_filter]
+
+            axis = tuple(
+                i for i in range(act_len) if (i != axis_filter) and (i != (len(shape_act) + axis_filter)))
+
+            dead_neurons = np.sum(np.sum(activation_values, axis=axis) == 0)
+
+            dead_neurons_share = float(dead_neurons) / float(total_featuremaps)
+            if (self.verbose and dead_neurons > 0) or dead_neurons_share >= self.dead_neurons_share_threshold:
+                str_warning = 'Layer {} (#{}) has {} dead neurons ({:.2%})!'.format(layer_name, layer_index,
+                                                                                    dead_neurons, dead_neurons_share)
+
+                print(str_warning)

tests/keras_contrib/backend/backend_test.py

@@ -157,8 +157,9 @@ class TestBackend(object):
                     th_var_val = KTH.eval(th_var)
                     tf_var_val = KTF.eval(tf_var)
 
-                    assert_allclose(th_mean_val, tf_mean_val, rtol=1e-4)
-                    assert_allclose(th_var_val, tf_var_val, rtol=1e-4)
+                    # absolute tolerance needed when working with zeros
+                    assert_allclose(th_mean_val, tf_mean_val, rtol=1e-4, atol=1e-10)
+                    assert_allclose(th_var_val, tf_var_val, rtol=1e-4, atol=1e-10)
 
     def test_clip(self):
         check_single_tensor_operation('clip', (4, 2), min_value=0.4, max_value=0.6)

tests/keras_contrib/callbacks/dead_relu_detector_test.py

@@ -1,40 +1,191 @@
 import pytest
-import warnings
 import numpy as np
+import sys
+
+if (sys.version_info > (3, 0)):
+    from io import StringIO
+else:
+    from StringIO import StringIO
 
 from keras_contrib import callbacks
 from keras.models import Sequential
-from keras.layers import Dense
+from keras.layers import Dense, Conv2D, Flatten
+from keras import backend as K
+
+n_out = 11  # with 1 neuron dead, 1/11 is just below the threshold of 10% with verbose = False
+
+
+def check_print(do_train, expected_warnings, nr_dead=None, perc_dead=None):
+    """
+    Receive stdout to check if correct warning message is delivered
+    :param nr_dead: int
+    :param perc_dead: float, 10% should be written as 0.1
+    """
+
+    saved_stdout = sys.stdout
+
+    out = StringIO()
+    out.flush()
+    sys.stdout = out    # overwrite current stdout
+
+    do_train()
+
+    stdoutput = out.getvalue().strip()  # get prints, can be something like: "Layer dense (#0) has 2 dead neurons (20.00%)!"
+    str_to_count = "dead neurons"
+    count = stdoutput.count(str_to_count)
+
+    sys.stdout = saved_stdout   # restore stdout
+    out.close()
+
+    assert expected_warnings == count
+    if expected_warnings and (nr_dead is not None):
+        str_to_check = 'has {} dead'.format(nr_dead)
+        assert str_to_check in stdoutput, '"{}" not in "{}"'.format(str_to_check, stdoutput)
+    if expected_warnings and (perc_dead is not None):
+        str_to_check = 'neurons ({:.2%})!'.format(perc_dead)
+        assert str_to_check in stdoutput, '"{}" not in "{}"'.format(str_to_check, stdoutput)
 
 
 def test_DeadDeadReluDetector():
-    def do_test(weights, expected_warnings, verbose):
-        with warnings.catch_warnings(record=True) as w:
-            dataset = np.ones((1, 1, 1))    # data to be fed as training
+    n_samples = 9
+
+    input_shape = (n_samples, 3, 4)  # 4 input features
+    shape_out = (n_samples, 3, n_out)  # 11 output features
+    shape_weights = (4, n_out)
+
+    # ignore batch size
+    input_shape_dense = tuple(input_shape[1:])
+
+    def do_test(weights, expected_warnings, verbose, nr_dead=None, perc_dead=None):
+
+        def do_train():
+            dataset = np.ones(input_shape)    # data to be fed as training
             model = Sequential()
-            model.add(Dense(10, activation='relu', input_shape=(1, 1), use_bias=False, weights=[weights]))
+            model.add(Dense(n_out, activation='relu', input_shape=input_shape_dense,
+                            use_bias=False, weights=[weights], name='dense'))
             model.compile(optimizer='sgd', loss='categorical_crossentropy')
             model.fit(
                 dataset,
-                np.ones((1, 1, 10)),
+                np.ones(shape_out),
+                batch_size=1,
                 epochs=1,
                 callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)],
                 verbose=False
             )
-            assert len(w) == expected_warnings
-            for warn_item in w:
-                assert issubclass(warn_item.category, RuntimeWarning)
-                assert "dead neurons" in str(warn_item.message)
 
-    weights_1_dead = np.ones((1, 10))      # weights that correspond to NN with 1/10 neurons dead
+        check_print(do_train, expected_warnings, nr_dead, perc_dead)
+
+    weights_1_dead = np.ones(shape_weights)  # weights that correspond to NN with 1/11 neurons dead
+    weights_2_dead = np.ones(shape_weights)  # weights that correspond to NN with 2/11 neurons dead
+    weights_all_dead = np.zeros(shape_weights)  # weights that correspond to all neurons dead
+
     weights_1_dead[:, 0] = 0
-    weights_2_dead = np.ones((1, 10))      # weights that correspond to NN with 2/10 neurons dead
-    weights_2_dead[:, 0] = 0
-    weights_2_dead[:, 1] = 0
+    weights_2_dead[:, 0:2] = 0
 
-    do_test(weights_1_dead, verbose=True, expected_warnings=1)
+    do_test(weights_1_dead, verbose=True, expected_warnings=1, nr_dead=1, perc_dead=1. / n_out)
     do_test(weights_1_dead, verbose=False, expected_warnings=0)
-    do_test(weights_2_dead, verbose=True, expected_warnings=1)
+    do_test(weights_2_dead, verbose=True, expected_warnings=1, nr_dead=2, perc_dead=2. / n_out)
+    # do_test(weights_all_dead, verbose=True, expected_warnings=1, nr_dead=n_out, perc_dead=1.)
+
+
+def test_DeadDeadReluDetector_bias():
+    n_samples = 9
+
+    input_shape = (n_samples, 4)  # 4 input features
+    shape_weights = (4, n_out)
+    shape_bias = (n_out, )
+    shape_out = (n_samples, n_out)  # 11 output features
+
+    # ignore batch size
+    input_shape_dense = tuple(input_shape[1:])
+
+    def do_test(weights, bias, expected_warnings, verbose, nr_dead=None, perc_dead=None):
+
+        def do_train():
+            dataset = np.ones(input_shape)  # data to be fed as training
+            model = Sequential()
+            model.add(Dense(n_out, activation='relu', input_shape=input_shape_dense,
+                            use_bias=True, weights=[weights, bias], name='dense'))
+            model.compile(optimizer='sgd', loss='categorical_crossentropy')
+            model.fit(
+                dataset,
+                np.ones(shape_out),
+                batch_size=1,
+                epochs=1,
+                callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)],
+                verbose=False
+            )
+
+        check_print(do_train, expected_warnings, nr_dead, perc_dead)
+
+    weights_1_dead = np.ones(shape_weights)  # weights that correspond to NN with 1/11 neurons dead
+    weights_2_dead = np.ones(shape_weights)  # weights that correspond to NN with 2/11 neurons dead
+    weights_all_dead = np.zeros(shape_weights)  # weights that correspond to all neurons dead
+
+    weights_1_dead[:, 0] = 0
+    weights_2_dead[:, 0:2] = 0
+
+    bias = np.zeros(shape_bias)
+
+    do_test(weights_1_dead, bias, verbose=True, expected_warnings=1, nr_dead=1, perc_dead=1. / n_out)
+    do_test(weights_1_dead, bias, verbose=False, expected_warnings=0)
+    do_test(weights_2_dead, bias, verbose=True, expected_warnings=1, nr_dead=2, perc_dead=2. / n_out)
+    # do_test(weights_all_dead, bias, verbose=True, expected_warnings=1, nr_dead=n_out, perc_dead=1.)
+
+
+def test_DeadDeadReluDetector_conv():
+    n_samples = 9
+
+    # (5, 5) kernel, 4 input featuremaps and 11 output featuremaps
+    if K.image_data_format() == 'channels_last':
+        input_shape = (n_samples, 5, 5, 4)
+    else:
+        input_shape = (n_samples, 4, 5, 5)
+
+    # ignore batch size
+    input_shape_conv = tuple(input_shape[1:])
+    shape_weights = (5, 5, 4, n_out)
+    shape_out = (n_samples, n_out)
+
+    def do_test(weights_bias, expected_warnings, verbose, nr_dead=None, perc_dead=None):
+        """
+        :param perc_dead: as float, 10% should be written as 0.1
+        """
+
+        def do_train():
+            dataset = np.ones(input_shape)  # data to be fed as training
+            model = Sequential()
+            model.add(Conv2D(n_out, (5, 5), activation='relu', input_shape=input_shape_conv,
+                             use_bias=True, weights=weights_bias, name='conv'))
+            model.add(Flatten())  # to handle Theano's categorical crossentropy
+            model.compile(optimizer='sgd', loss='categorical_crossentropy')
+            model.fit(
+                dataset,
+                np.ones(shape_out),
+                batch_size=1,
+                epochs=1,
+                callbacks=[callbacks.DeadReluDetector(dataset, verbose=verbose)],
+                verbose=False
+            )
+
+        check_print(do_train, expected_warnings, nr_dead, perc_dead)
+
+    weights_1_dead = np.ones(shape_weights)      # weights that correspond to NN with 1/11 neurons dead
+    weights_1_dead[..., 0] = 0
+    weights_2_dead = np.ones(shape_weights)    # weights that correspond to NN with 2/11 neurons dead
+    weights_2_dead[..., 0:2] = 0
+    weights_all_dead = np.zeros(shape_weights)    # weights that correspond to NN with all neurons dead
+
+    bias = np.zeros((11, ))
+
+    weights_bias_1_dead = [weights_1_dead, bias]
+    weights_bias_2_dead = [weights_2_dead, bias]
+    weights_bias_all_dead = [weights_all_dead, bias]
+
+    do_test(weights_bias_1_dead, verbose=True, expected_warnings=1, nr_dead=1, perc_dead=1. / n_out)
+    do_test(weights_bias_1_dead, verbose=False, expected_warnings=0)
+    do_test(weights_bias_2_dead, verbose=True, expected_warnings=1, nr_dead=2, perc_dead=2. / n_out)
+    # do_test(weights_bias_all_dead, verbose=True, expected_warnings=1, nr_dead=n_out, perc_dead=1.)
 
 
 if __name__ == '__main__':