wassname/keras-contrib
1
0
Fork 0
mirror of https://github.com/wassname/keras-contrib.git synced 2026-06-27 16:10:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

add regularizers and fix example

Browse Source
This commit is contained in:
Somshubra Majumdar
2017-12-04 13:02:21 -06:00
parent 9f7ab69249
commit 0fad550fed
2 changed files with 76 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/cifar10_nasnet.py
+5 -8
View File
@@ -18,7 +18,7 @@ import numpy as np
weights_file = 'NASNet-CIFAR-10.h5'
lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.5), cooldown=0, patience=5, min_lr=0.5e-6)
lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.5), cooldown=0, patience=5, min_lr=0.5e-5)
csv_logger = CSVLogger('NASNet-CIFAR-10.csv')
model_checkpoint = ModelCheckpoint(weights_file, monitor='val_predictions_acc', save_best_only=True,
save_weights_only=True, mode='max')
@@ -51,12 +51,8 @@ X_train /= 128.
X_test /= 128.
# For training, the auxilary branch must be used to correctly train NASNet
model = NASNetCIFAR((img_rows, img_cols, img_channels), dropout=0.5,
use_auxilary_branch=True)
model.compile(loss=['categorical_crossentropy', 'categorical_crossentropy'],
optimizer='adam',
loss_weights=[1.0, 0.4],
metrics=['accuracy'])
model = NASNetCIFAR((img_rows, img_cols, img_channels))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
if not data_augmentation:
print('Not using data augmentation.')
@@ -65,6 +61,7 @@ if not data_augmentation:
nb_epoch=nb_epoch,
validation_data=(X_test, Y_test),
shuffle=True,
verbose=1,
callbacks=[lr_reducer, csv_logger, model_checkpoint])
else:
print('Using real-time data augmentation.')
@@ -89,7 +86,7 @@ else:
model.fit_generator(datagen.flow(X_train, Y_train, batch_size=batch_size),
steps_per_epoch=X_train.shape[0] // batch_size,
validation_data=(X_test, Y_test),
epochs=nb_epoch, verbose=2,
epochs=nb_epoch, verbose=1,
callbacks=[lr_reducer, csv_logger, model_checkpoint])
scores = model.evaluate(X_test, Y_test, batch_size=batch_size)
keras_contrib/applications/nasnet.py
+71 -34
View File
@@ -33,6 +33,7 @@ from keras.layers import ZeroPadding2D
from keras.layers import Cropping2D
from keras.layers import concatenate
from keras.layers import add
from keras.regularizers import l2
from keras.utils.data_utils import get_file
from keras.engine.topology import get_source_inputs
from keras.applications.imagenet_utils import _obtain_input_shape
@@ -52,6 +53,7 @@ def NASNet(input_shape=None,
use_auxilary_branch=False,
filters_multiplier=2,
dropout=0.5,
weight_decay=5e-5,
include_top=True,
weights=None,
input_tensor=None,
@@ -93,6 +95,7 @@ def NASNet(input_shape=None,
- If `filters_multiplier` = 1, default number of filters from the paper
are used at each layer.
dropout: dropout rate
weight_decay: l2 regularization weight
include_top: whether to include the fully-connected
layer at the top of the network.
weights: `None` (random initialization) or
@@ -178,7 +181,7 @@ def NASNet(input_shape=None,
filters = penultimate_filters // 24
x = Conv2D(stem_filters, (3, 3), strides=(2, 2), padding='valid', use_bias=False, name='stem_conv1',
kernel_initializer='he_normal')(img_input)
kernel_initializer='he_normal', kernel_regularizer=l2(weight_decay))(img_input)
x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON,
name='stem_bn1')(x)
@@ -204,20 +207,21 @@ def NASNet(input_shape=None,
auxilary_x = Activation('relu')(x)
auxilary_x = AveragePooling2D((5, 5), strides=(3, 3), padding='valid', name='aux_pool')(auxilary_x)
auxilary_x = Conv2D(128, (1, 1), padding='same', use_bias=False, name='aux_conv_projection',
kernel_initializer='he_normal')(auxilary_x)
kernel_initializer='he_normal', kernel_regularizer=l2(weight_decay))(auxilary_x)
auxilary_x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON,
name='aux_bn_projection')(auxilary_x)
auxilary_x = Activation('relu')(auxilary_x)
auxilary_x = Conv2D(768, (auxilary_x._keras_shape[img_height], auxilary_x._keras_shape[img_width]),
padding='valid', use_bias=False, kernel_initializer='he_normal',
name='aux_conv_reduction')(auxilary_x)
kernel_regularizer=l2(weight_decay), name='aux_conv_reduction')(auxilary_x)
auxilary_x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON,
name='aux_bn_reduction')(auxilary_x)
auxilary_x = Activation('relu')(auxilary_x)
auxilary_x = GlobalAveragePooling2D()(auxilary_x)
auxilary_x = Dense(classes, activation='softmax', name='aux_predictions')(auxilary_x)
auxilary_x = Dense(classes, activation='softmax', kernel_regularizer=l2(weight_decay),
name='aux_predictions')(auxilary_x)
x, p0 = _reduction_A(x, p, filters * filters_multiplier ** 2, id='reduce_%d' % (2 * nb_blocks))
@@ -231,7 +235,7 @@ def NASNet(input_shape=None,
if include_top:
x = GlobalAveragePooling2D()(x)
x = Dropout(dropout)(x)
x = Dense(classes, activation='softmax')(x)
x = Dense(classes, activation='softmax', kernel_regularizer=l2(weight_decay), name='predictions')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
@@ -252,7 +256,8 @@ def NASNet(input_shape=None,
model = Model(inputs, x, name='NASNet')
# load weights (when available)
warnings.warn('Weights of NASNet models have not been ported yet for Keras.')
if weights is not None:
warnings.warn('Weights of NASNet models have not been ported yet for Keras.')
if old_data_format:
K.set_image_data_format(old_data_format)
@@ -260,11 +265,12 @@ def NASNet(input_shape=None,
return model
def NASNetLarge(input_shape=None,
def NASNetLarge(input_shape=(331, 331, 3),
dropout=0.5,
weight_decay=5e-5,
use_auxilary_branch=False,
include_top=True,
weights='imagenet',
weights=None,
input_tensor=None,
pooling=None,
classes=1000):
@@ -284,6 +290,7 @@ def NASNetLarge(input_shape=None,
use_auxilary_branch: Whether to use the auxilary branch during
training or evaluation.
dropout: dropout rate
weight_decay: l2 regularization weight
include_top: whether to include the fully-connected
layer at the top of the network.
weights: `None` (random initialization) or
@@ -315,6 +322,10 @@ def NASNetLarge(input_shape=None,
RuntimeError: If attempting to run this model with a
backend that does not support separable convolutions.
"""
global _BN_DECAY, _BN_EPSILON
_BN_DECAY = 0.9997
_BN_EPSILON = 1e-3
return NASNet(input_shape,
penultimate_filters=4032,
nb_blocks=6,
@@ -323,6 +334,7 @@ def NASNetLarge(input_shape=None,
use_auxilary_branch=use_auxilary_branch,
filters_multiplier=2,
dropout=dropout,
weight_decay=weight_decay,
include_top=include_top,
weights=weights,
input_tensor=input_tensor,
@@ -331,11 +343,12 @@ def NASNetLarge(input_shape=None,
default_size=331)
def NASNetMobile(input_shape=None,
def NASNetMobile(input_shape=(224, 224, 3),
dropout=0.5,
weight_decay=4e-5,
use_auxilary_branch=False,
include_top=True,
weights='imagenet',
weights=None,
input_tensor=None,
pooling=None,
classes=1000):
@@ -355,6 +368,7 @@ def NASNetMobile(input_shape=None,
use_auxilary_branch: Whether to use the auxilary branch during
training or evaluation.
dropout: dropout rate
weight_decay: l2 regularization weight
include_top: whether to include the fully-connected
layer at the top of the network.
weights: `None` (random initialization) or
@@ -386,6 +400,10 @@ def NASNetMobile(input_shape=None,
RuntimeError: If attempting to run this model with a
backend that does not support separable convolutions.
"""
global _BN_DECAY, _BN_EPSILON
_BN_DECAY = 0.9997
_BN_EPSILON = 1e-3
return NASNet(input_shape,
penultimate_filters=1056,
nb_blocks=4,
@@ -394,6 +412,7 @@ def NASNetMobile(input_shape=None,
use_auxilary_branch=use_auxilary_branch,
filters_multiplier=2,
dropout=dropout,
weight_decay=weight_decay,
include_top=include_top,
weights=weights,
input_tensor=input_tensor,
@@ -402,8 +421,9 @@ def NASNetMobile(input_shape=None,
default_size=224)
def NASNetCIFAR(input_shape=None,
def NASNetCIFAR(input_shape=(32, 32, 3),
dropout=0.0,
weight_decay=5e-4,
use_auxilary_branch=False,
include_top=True,
weights=None,
@@ -426,6 +446,7 @@ def NASNetCIFAR(input_shape=None,
use_auxilary_branch: Whether to use the auxilary branch during
training or evaluation.
dropout: dropout rate
weight_decay: l2 regularization weight
include_top: whether to include the fully-connected
layer at the top of the network.
weights: `None` (random initialization) or
@@ -457,6 +478,10 @@ def NASNetCIFAR(input_shape=None,
RuntimeError: If attempting to run this model with a
backend that does not support separable convolutions.
"""
global _BN_DECAY, _BN_EPSILON
_BN_DECAY = 0.9
_BN_EPSILON = 1e-5
return NASNet(input_shape,
penultimate_filters=768,
nb_blocks=6,
@@ -465,6 +490,7 @@ def NASNetCIFAR(input_shape=None,
use_auxilary_branch=use_auxilary_branch,
filters_multiplier=2,
dropout=dropout,
weight_decay=weight_decay,
include_top=include_top,
weights=weights,
input_tensor=input_tensor,
@@ -473,7 +499,7 @@ def NASNetCIFAR(input_shape=None,
default_size=224)
def _separable_conv_block(ip, filters, kernel_size=(3, 3), strides=(1, 1), id=None):
def _separable_conv_block(ip, filters, kernel_size=(3, 3), strides=(1, 1), weight_decay=5e-5, id=None):
'''Adds 2 blocks of [relu-separable conv-batchnorm]
# Arguments:
@@ -481,6 +507,7 @@ def _separable_conv_block(ip, filters, kernel_size=(3, 3), strides=(1, 1), id=No
filters: number of output filters per layer
kernel_size: kernel size of separable convolutions
strides: strided convolution for downsampling
weight_decay: l2 regularization weight
id: string id
# Returns:
@@ -491,18 +518,20 @@ def _separable_conv_block(ip, filters, kernel_size=(3, 3), strides=(1, 1), id=No
with K.name_scope('separable_conv_block_%s' % id):
x = Activation('relu')(ip)
x = SeparableConv2D(filters, kernel_size, strides=strides, name='separable_conv_1_%s' % id,
padding='same', use_bias=False, kernel_initializer='he_normal')(x)
padding='same', use_bias=False, kernel_initializer='he_normal',
kernel_regularizer=l2(weight_decay))(x)
x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON,
name="separable_conv_1_bn_%s" % (id))(x)
x = Activation('relu')(x)
x = SeparableConv2D(filters, kernel_size, name='separable_conv_2_%s' % id,
padding='same', use_bias=False, kernel_initializer='he_normal')(x)
padding='same', use_bias=False, kernel_initializer='he_normal',
kernel_regularizer=l2(weight_decay))(x)
x = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON,
name="separable_conv_2_bn_%s" % (id))(x)
return x
def _adjust_block(p, ip, filters, id=None):
def _adjust_block(p, ip, filters, weight_decay=5e-5, id=None):
'''
Adjusts the input `p` to match the shape of the `input`
or situations where the output number of filters needs to
@@ -512,6 +541,7 @@ def _adjust_block(p, ip, filters, id=None):
p: input tensor which needs to be modified
ip: input tensor whose shape needs to be matched
filters: number of output filters to be matched
weight_decay: l2 regularization weight
id: string id
# Returns:
@@ -529,13 +559,13 @@ def _adjust_block(p, ip, filters, id=None):
p = Activation('relu', name='adjust_relu_1_%s' % id)(p)
p1 = AveragePooling2D((1, 1), strides=(2, 2), padding='valid', name='adjust_avg_pool_1_%s' % id)(p)
p1 = Conv2D(filters // 2, (1, 1), padding='same', use_bias=False,
p1 = Conv2D(filters // 2, (1, 1), padding='same', use_bias=False, kernel_regularizer=l2(weight_decay),
name='adjust_conv_1_%s' % id, kernel_initializer='he_normal')(p1)
p2 = ZeroPadding2D(padding=((0, 1), (0, 1)))(p)
p2 = Cropping2D(cropping=((1, 0), (1, 0)))(p2)
p2 = AveragePooling2D((1, 1), strides=(2, 2), padding='valid', name='adjust_avg_pool_2_%s' % id)(p2)
p2 = Conv2D(filters // 2, (1, 1), padding='same', use_bias=False,
p2 = Conv2D(filters // 2, (1, 1), padding='same', use_bias=False, kernel_regularizer=l2(weight_decay),
name='adjust_conv_2_%s' % id, kernel_initializer='he_normal')(p2)
p = concatenate([p1, p2], axis=channel_dim)
@@ -546,19 +576,20 @@ def _adjust_block(p, ip, filters, id=None):
with K.name_scope('adjust_projection_block_%s' % id):
p = Activation('relu')(p)
p = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='adjust_conv_projection_%s' % id,
use_bias=False, kernel_initializer='he_normal')(p)
use_bias=False, kernel_regularizer=l2(weight_decay), kernel_initializer='he_normal')(p)
p = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON,
name='adjust_bn_%s' % id)(p)
return p
def _normal_A(ip, p, filters, id=None):
def _normal_A(ip, p, filters, weight_decay=5e-5, id=None):
'''Adds a Normal cell for NASNet-A (Fig. 4 in the paper)
# Arguments:
ip: input tensor `x`
p: input tensor `p`
filters: number of output filters
weight_decay: l2 regularization weight
id: string id
# Returns:
@@ -567,21 +598,22 @@ def _normal_A(ip, p, filters, id=None):
channel_dim = 1 if K.image_data_format() == 'channels_first' else -1
with K.name_scope('normal_A_block_%s' % id):
p = _adjust_block(p, ip, filters, id)
p = _adjust_block(p, ip, filters, weight_decay, id)
h = Activation('relu')(ip)
h = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='normal_conv_1_%s' % id,
use_bias=False, kernel_initializer='he_normal')(h)
use_bias=False, kernel_initializer='he_normal', kernel_regularizer=l2(weight_decay))(h)
h = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON,
name='normal_bn_1_%s' % id)(h)
with K.name_scope('block_1'):
x1 = _separable_conv_block(h, filters, id='normal_left1_%s' % id)
x1 = _separable_conv_block(h, filters, weight_decay=weight_decay, id='normal_left1_%s' % id)
x1 = add([x1, h], name='normal_add_1_%s' % id)
with K.name_scope('block_2'):
x2_1 = _separable_conv_block(p, filters, id='normal_left2_%s' % id)
x2_2 = _separable_conv_block(h, filters, kernel_size=(5, 5), id='normal_right2_%s' % id)
x2_1 = _separable_conv_block(p, filters, weight_decay=weight_decay, id='normal_left2_%s' % id)
x2_2 = _separable_conv_block(h, filters, kernel_size=(5, 5), weight_decay=weight_decay,
id='normal_right2_%s' % id)
x2 = add([x2_1, x2_2], name='normal_add_2_%s' % id)
with K.name_scope('block_3'):
@@ -594,21 +626,22 @@ def _normal_A(ip, p, filters, id=None):
x4 = add([x4_1, x4_2], name='normal_add_4_%s' % id)
with K.name_scope('block_5'):
x5_1 = _separable_conv_block(p, filters, (5, 5), id='normal_left5_%s' % id)
x5_2 = _separable_conv_block(p, filters, (3, 3), id='normal_right5_%s' % id)
x5_1 = _separable_conv_block(p, filters, (5, 5), weight_decay=weight_decay, id='normal_left5_%s' % id)
x5_2 = _separable_conv_block(p, filters, (3, 3), weight_decay=weight_decay, id='normal_right5_%s' % id)
x5 = add([x5_1, x5_2], name='normal_add_5_%s' % id)
x = concatenate([p, x2, x5, x3, x4, x1], axis=channel_dim, name='normal_concat_%s' % id)
return x, ip
def _reduction_A(ip, p, filters, id=None):
def _reduction_A(ip, p, filters, weight_decay=5e-5, id=None):
'''Adds a Reduction cell for NASNet-A (Fig. 4 in the paper)
# Arguments:
ip: input tensor `x`
p: input tensor `p`
filters: number of output filters
weight_decay: l2 regularization weight
id: string id
# Returns:
@@ -618,32 +651,36 @@ def _reduction_A(ip, p, filters, id=None):
channel_dim = 1 if K.image_data_format() == 'channels_first' else -1
with K.name_scope('reduction_A_block_%s' % id):
p = _adjust_block(p, ip, filters, id)
p = _adjust_block(p, ip, filters, weight_decay, id)
h = Activation('relu')(ip)
h = Conv2D(filters, (1, 1), strides=(1, 1), padding='same', name='reduction_conv_1_%s' % id,
use_bias=False, kernel_initializer='he_normal')(h)
use_bias=False, kernel_initializer='he_normal', kernel_regularizer=l2(weight_decay))(h)
h = BatchNormalization(axis=channel_dim, momentum=_BN_DECAY, epsilon=_BN_EPSILON,
name='reduction_bn_1_%s' % id)(h)
with K.name_scope('block_1'):
x1_1 = _separable_conv_block(p, filters, (7, 7), strides=(2, 2), id='reduction_left1_%s' % id)
x1_2 = _separable_conv_block(h, filters, (5, 5), strides=(2, 2), id='reduction_right1_%s' % id)
x1_1 = _separable_conv_block(p, filters, (7, 7), strides=(2, 2), weight_decay=weight_decay,
id='reduction_left1_%s' % id)
x1_2 = _separable_conv_block(h, filters, (5, 5), strides=(2, 2), weight_decay=weight_decay,
id='reduction_right1_%s' % id)
x1 = add([x1_1, x1_2], name='reduction_add_1_%s' % id)
with K.name_scope('block_2'):
x2_1 = MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left2_%s' % id)(h)
x2_2 = _separable_conv_block(p, filters, (7, 7), strides=(2, 2), id='reduction_right2_%s' % id)
x2_2 = _separable_conv_block(p, filters, (7, 7), strides=(2, 2), weight_decay=weight_decay,
id='reduction_right2_%s' % id)
x2 = add([x2_1, x2_2], name='reduction_add_2_%s' % id)
with K.name_scope('block_3'):
x3_1 = AveragePooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left3_%s' % id)(h)
x3_2 = _separable_conv_block(p, filters, (5, 5), strides=(2, 2), id='reduction_right3_%s' % id)
x3_2 = _separable_conv_block(p, filters, (5, 5), strides=(2, 2), weight_decay=weight_decay,
id='reduction_right3_%s' % id)
x3 = add([x3_1, x3_2], name='reduction_add3_%s' % id)
with K.name_scope('block_4'):
x4_1 = MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='reduction_left4_%s' % id)(h)
x4_2 = _separable_conv_block(x1, filters, (3, 3), id='reduction_right4_%s' % id)
x4_2 = _separable_conv_block(x1, filters, (3, 3), weight_decay=weight_decay, id='reduction_right4_%s' % id)
x4 = add([x4_1, x4_2], name='reduction_add4_%s' % id)
with K.name_scope('block_5'):