From cb640bfe690f9bd8478513327897c08cde067a9d Mon Sep 17 00:00:00 2001 From: codekansas Date: Tue, 26 Jul 2016 10:21:06 -0700 Subject: [PATCH] refactored some stuff --- attention_lstm.py | 8 +++--- insurance_qa_eval.py | 41 +++++++++++------------------- keras_models.py | 60 +++++++++++++++++++------------------------- 3 files changed, 43 insertions(+), 66 deletions(-) diff --git a/attention_lstm.py b/attention_lstm.py index 1724e81..03de71c 100644 --- a/attention_lstm.py +++ b/attention_lstm.py @@ -6,11 +6,9 @@ from keras.layers import LSTM, activations class AttentionLSTM(LSTM): def __init__(self, output_dim, attention_vec, attn_activation='tanh', - attn_inner_activation='tanh', single_attention_param=False, - n_attention_dim=None, **kwargs): + single_attention_param=False, n_attention_dim=None, **kwargs): self.attention_vec = attention_vec self.attn_activation = activations.get(attn_activation) - self.attn_inner_activation = activations.get(attn_inner_activation) self.single_attention_param = single_attention_param self.n_attention_dim = output_dim if n_attention_dim is None else n_attention_dim @@ -51,10 +49,10 @@ class AttentionLSTM(LSTM): h, [h, c] = super(AttentionLSTM, self).step(x, states) attention = states[4] - m = self.attn_inner_activation(K.dot(h, self.U_a) * attention + self.b_a) + m = self.attn_activation(K.dot(h, self.U_a) * attention + self.b_a) # Intuitively it makes more sense to use a sigmoid (was getting some NaN problems # which I think might have been caused by the exponential function -> gradients blow up) - s = self.attn_activation(K.dot(m, self.U_s) + self.b_s) + s = K.sigmoid(K.dot(m, self.U_s) + self.b_s) if self.single_attention_param: h = h * K.repeat_elements(s, self.output_dim, axis=1) diff --git a/insurance_qa_eval.py b/insurance_qa_eval.py index 35f0775..5a568c8 100644 --- a/insurance_qa_eval.py +++ b/insurance_qa_eval.py @@ -8,9 +8,10 @@ from time import strftime, gmtime import pickle +from keras.optimizers import Adam from scipy.stats import rankdata -from keras_models import EmbeddingModel, AttentionModel +from keras_models import EmbeddingModel, AttentionModel, ConvolutionModel random.seed(42) @@ -104,20 +105,17 @@ class Evaluator: questions = self.padq(questions) good_answers = self.pada(good_answers) - # bad_answers = self.pada(random.sample(self.answers.values(), len(good_answers))) val_loss = {'loss': 1., 'epoch': 0} for i in range(1, nb_epoch): - # bad_answers = np.roll(good_answers, random.randint(10, len(questions) - 10)) - # bad_answers = good_answers.copy() - # random.shuffle(bad_answers) + # sample from all answers to get bad answers bad_answers = self.pada(random.sample(self.answers.values(), len(good_answers))) # shuffle questions - # zipped = zip(questions, good_answers) - # random.shuffle(zipped) - # questions[:], good_answers[:] = zip(*zipped) + zipped = zip(questions, good_answers) + random.shuffle(zipped) + questions[:], good_answers[:] = zip(*zipped) print('Epoch %d :: ' % i, end='') self.print_time() @@ -128,9 +126,6 @@ class Evaluator: val_loss = {'loss': hist.history['val_loss'][0], 'epoch': i} print('Best: Loss = {}, Epoch = {}'.format(val_loss['loss'], val_loss['epoch'])) - if eval_every is not None and i % eval_every == 0: - self.get_mrr(model) - if save_every is not None and i % save_every == 0: self.save_epoch(model, i) @@ -227,25 +222,17 @@ if __name__ == '__main__': import numpy as np conf = { - 'question_len': 20, - 'answer_len': 60, + 'question_len': 200, + 'answer_len': 200, 'n_words': 22353, # len(vocabulary) + 1 'margin': 0.2, 'training_params': { 'save_every': 1, - # 'eval_every': 1, 'batch_size': 20, - 'nb_epoch': 1000, + 'nb_epoch': 100, 'validation_split': 0.2, - 'optimizer': 'adam', - # 'optimizer': Adam(clip_norm=0.1), - # 'n_eval': 100, - - 'evaluate_all_threshold': { - 'mode': 'all', - 'top1': 0.4, - }, + 'optimizer': Adam(clipnorm=1e-2), }, 'model_params': { @@ -254,7 +241,7 @@ if __name__ == '__main__': # convolution 'nb_filters': 1000, # * 4 - 'conv_activation': 'relu', + 'conv_activation': 'tanh', # recurrent 'n_lstm_dims': 141, # * 2 @@ -263,7 +250,7 @@ if __name__ == '__main__': }, 'similarity_params': { - 'mode': 'cosine', + 'mode': 'gesd', 'gamma': 1, 'c': 1, 'd': 2, @@ -284,12 +271,12 @@ if __name__ == '__main__': # np.save(open('models/embedding_1000_dim.h5', 'wb'), weights) # train the model - # evaluator.load_epoch(model, 54) + # evaluator.load_epoch(model, 42) best_loss = evaluator.train(model) # evaluate mrr for a particular epoch evaluator.load_epoch(model, best_loss['epoch']) - # evaluator.load_epoch(model, 116) + # evaluator.load_epoch(model, 31) evaluator.get_mrr(model, evaluate_all=True) # for epoch in range(1, 100): # print('Epoch %d' % epoch) diff --git a/keras_models.py b/keras_models.py index d07bd5a..07d44cf 100644 --- a/keras_models.py +++ b/keras_models.py @@ -118,7 +118,7 @@ class LanguageModel: bad_output = qa_model([self.question, self.answer_bad]) loss = merge([good_output, bad_output], - mode=lambda x: K.maximum(1e-6, self.config['margin'] - x[0] + x[1]), + mode=lambda x: K.relu(self.config['margin'] - x[0] + x[1]), output_shape=lambda x: x[0]) self.training_model = Model(input=[self.question, self.answer_good, self.answer_bad], output=loss) @@ -201,24 +201,24 @@ class ConvolutionModel(LanguageModel): # embedding.updates = [] # dropout - dropout = Dropout(0.25) + dropout = Dropout(0.5) question_dropout = dropout(question_embedding) answer_dropout = dropout(answer_embedding) # dense - dense = TimeDistributed(Dense(self.model_params.get('n_hidden', 200), activation='tanh', - activity_regularizer=regularizers.activity_l1(1e-4), - W_regularizer=regularizers.l1(1e-4), - dropout=0.5)) - question_dense = dense(question_dropout) - answer_dense = dense(answer_dropout) + dense = TimeDistributed(Dense(self.model_params.get('n_hidden', 200), + # activity_regularizer=regularizers.activity_l1(1e-4), + # W_regularizer=regularizers.l1(1e-4), + activation='tanh')) + question_dense = dropout(dense(question_dropout)) + answer_dense = dropout(dense(answer_dropout)) # cnn cnns = [Convolution1D(filter_length=filter_length, nb_filter=self.model_params.get('nb_filters', 1000), activation=self.model_params.get('conv_activation', 'relu'), - W_regularizer=regularizers.l1(1e-4), - activity_regularizer=regularizers.activity_l1(1e-4), + # W_regularizer=regularizers.l1(1e-4), + # activity_regularizer=regularizers.activity_l1(1e-4), border_mode='same') for filter_length in [2, 3, 5, 7]] question_cnn = merge([cnn(question_dense) for cnn in cnns], mode='concat') answer_cnn = merge([cnn(answer_dense) for cnn in cnns], mode='concat') @@ -229,14 +229,15 @@ class ConvolutionModel(LanguageModel): # maxpooling maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) + avepool = Lambda(lambda x: K.mean(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) question_pool = maxpool(question_dropout) answer_pool = maxpool(answer_dropout) # activation - dropout = Dropout(0.4) + larger_dropout = Dropout(0.5) activation = Activation('linear') - question_output = activation(dropout(question_pool)) - answer_output = activation(dropout(answer_pool)) + question_output = larger_dropout(activation(question_pool)) + answer_output = larger_dropout(activation(answer_pool)) return question_output, answer_output @@ -260,39 +261,30 @@ class AttentionModel(LanguageModel): # embedding.params = [] # embedding.updates = [] - # dropout - dropout = Dropout(0.25) - question_dropout = dropout(question_embedding) - answer_dropout = dropout(answer_embedding) - # question rnn part f_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True, dropout_U=0.2, - U_regularizer=regularizers.l2(1e-4), consume_less='mem') + consume_less='mem') b_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True, dropout_U=0.2, - U_regularizer=regularizers.l2(1e-4), consume_less='mem', go_backwards=True) - question_f_rnn = f_rnn(question_dropout) - question_b_rnn = b_rnn(question_dropout) - question_f_dropout = dropout(question_f_rnn) - question_b_dropout = dropout(question_b_rnn) + consume_less='mem', go_backwards=True) + question_f_rnn = f_rnn(question_embedding) + question_b_rnn = b_rnn(question_embedding) # maxpooling maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) avepool = Lambda(lambda x: K.mean(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) - question_pool = merge([avepool(question_f_dropout), avepool(question_b_dropout)], mode='concat', concat_axis=-1) + question_pool = merge([maxpool(question_f_rnn), maxpool(question_b_rnn)], mode='concat', concat_axis=-1) # answer rnn part - f_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, dropout_U=0.2, - U_regularizer=regularizers.l1(1e-4), return_sequences=True, consume_less='mem', - single_attention_param=True) - b_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, dropout_U=0.2, - U_regularizer=regularizers.l1(1e-4), return_sequences=True, consume_less='mem', - go_backwards=True, single_attention_param=True) - answer_f_rnn = f_rnn(answer_dropout) - answer_b_rnn = b_rnn(answer_dropout) + f_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, return_sequences=True, + consume_less='mem', single_attention_param=True) + b_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, return_sequences=True, + consume_less='mem', go_backwards=True, single_attention_param=True) + answer_f_rnn = f_rnn(answer_embedding) + answer_b_rnn = b_rnn(answer_embedding) answer_pool = merge([maxpool(answer_f_rnn), maxpool(answer_b_rnn)], mode='concat', concat_axis=-1) # activation - dropout = Dropout(0.4) + dropout = Dropout(0.5) activation = Activation('linear') question_output = activation(dropout(question_pool)) answer_output = activation(dropout(answer_pool))