diff --git a/insurance_qa_eval.py b/insurance_qa_eval.py index 5a568c8..5fdcdd5 100644 --- a/insurance_qa_eval.py +++ b/insurance_qa_eval.py @@ -26,7 +26,7 @@ class Evaluator: self.path = data_path self.conf = dict() if conf is None else conf self.params = conf.get('training_params', dict()) - self.answers = self.load('answers') # self.load('generated') + self.answers = self.load('answers') # self.load('generated') self._vocab = None self._reverse_vocab = None self._eval_sets = None @@ -88,7 +88,6 @@ class Evaluator: print(strftime('%Y-%m-%d %H:%M:%S :: ', gmtime()), end='') def train(self, model): - eval_every = self.params.get('eval_every', None) save_every = self.params.get('save_every', None) batch_size = self.params.get('batch_size', 128) nb_epoch = self.params.get('nb_epoch', 10) @@ -112,11 +111,6 @@ class Evaluator: # sample from all answers to get bad answers bad_answers = self.pada(random.sample(self.answers.values(), len(good_answers))) - # shuffle questions - zipped = zip(questions, good_answers) - random.shuffle(zipped) - questions[:], good_answers[:] = zip(*zipped) - print('Epoch %d :: ' % i, end='') self.print_time() hist = model.fit([questions, good_answers, bad_answers], nb_epoch=1, batch_size=batch_size, @@ -177,7 +171,7 @@ class Evaluator: max_r = np.argmax(r) max_n = np.argmax(r[:n_good]) - # print(' '.join(self.revert(d['question']))) + # print(' '.join(self.revegrt(d['question']))) # print(' '.join(self.revert(self.answers[indices[max_r]]))) # print(' '.join(self.revert(self.answers[indices[max_n]]))) @@ -222,10 +216,10 @@ if __name__ == '__main__': import numpy as np conf = { - 'question_len': 200, - 'answer_len': 200, + 'question_len': 50, + 'answer_len': 100, 'n_words': 22353, # len(vocabulary) + 1 - 'margin': 0.2, + 'margin': 0.009, 'training_params': { 'save_every': 1, @@ -247,6 +241,7 @@ if __name__ == '__main__': 'n_lstm_dims': 141, # * 2 'initial_embed_weights': np.load('models/word2vec_100_dim.h5'), + 'similarity_dropout': 0.5, }, 'similarity_params': { @@ -260,7 +255,7 @@ if __name__ == '__main__': evaluator = Evaluator(conf) ##### Define model ###### - model = AttentionModel(conf) + model = EmbeddingModel(conf) optimizer = conf.get('training_params', dict()).get('optimizer', 'adam') model.compile(optimizer=optimizer) @@ -271,12 +266,12 @@ if __name__ == '__main__': # np.save(open('models/embedding_1000_dim.h5', 'wb'), weights) # train the model - # evaluator.load_epoch(model, 42) + # evaluator.load_epoch(model, 6) best_loss = evaluator.train(model) # evaluate mrr for a particular epoch evaluator.load_epoch(model, best_loss['epoch']) - # evaluator.load_epoch(model, 31) + # evaluator.load_epoch(model, 68) evaluator.get_mrr(model, evaluate_all=True) # for epoch in range(1, 100): # print('Epoch %d' % epoch) diff --git a/keras_models.py b/keras_models.py index 07d44cf..665a552 100644 --- a/keras_models.py +++ b/keras_models.py @@ -117,7 +117,9 @@ class LanguageModel: good_output = qa_model([self.question, self.answer_good]) bad_output = qa_model([self.question, self.answer_bad]) - loss = merge([good_output, bad_output], + dropout = Dropout(self.model_params.get('similarity_dropout', 0.5)) + + loss = merge([dropout(good_output), dropout(bad_output)], mode=lambda x: K.relu(self.config['margin'] - x[0] + x[1]), output_shape=lambda x: x[0]) @@ -130,6 +132,9 @@ class LanguageModel: def fit(self, x, **kwargs): assert self.training_model is not None, 'Must compile the model before fitting data' y = np.zeros(shape=x[0].shape[:1]) + if 'validation_data' in kwargs: + data = kwargs['validation_data'] + kwargs['validation_data'] = [data, np.zeros(shape=data[0].shape[:1])] return self.training_model.fit(x, y, **kwargs) def predict(self, x, **kwargs): @@ -160,22 +165,12 @@ class EmbeddingModel(LanguageModel): question_embedding = embedding(question) answer_embedding = embedding(answer) - # dropout - dropout = Dropout(0.5) - question_dropout = dropout(question_embedding) - answer_dropout = dropout(answer_embedding) - # maxpooling maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) - question_maxpool = maxpool(question_dropout) - answer_maxpool = maxpool(answer_dropout) + question_pool = maxpool(question_embedding) + answer_pool = maxpool(answer_embedding) - # activation - activation = Activation('linear') - question_output = activation(question_maxpool) - answer_output = activation(answer_maxpool) - - return question_output, answer_output + return question_pool, answer_pool class ConvolutionModel(LanguageModel): @@ -200,18 +195,13 @@ class ConvolutionModel(LanguageModel): # embedding.params = [] # embedding.updates = [] - # dropout - dropout = Dropout(0.5) - question_dropout = dropout(question_embedding) - answer_dropout = dropout(answer_embedding) - # dense dense = TimeDistributed(Dense(self.model_params.get('n_hidden', 200), # activity_regularizer=regularizers.activity_l1(1e-4), # W_regularizer=regularizers.l1(1e-4), activation='tanh')) - question_dense = dropout(dense(question_dropout)) - answer_dense = dropout(dense(answer_dropout)) + question_dense = dense(question_embedding) + answer_dense = dense(answer_embedding) # cnn cnns = [Convolution1D(filter_length=filter_length, @@ -223,23 +213,13 @@ class ConvolutionModel(LanguageModel): question_cnn = merge([cnn(question_dense) for cnn in cnns], mode='concat') answer_cnn = merge([cnn(answer_dense) for cnn in cnns], mode='concat') - # dropout - question_dropout = dropout(question_cnn) - answer_dropout = dropout(answer_cnn) - # maxpooling maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) avepool = Lambda(lambda x: K.mean(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) - question_pool = maxpool(question_dropout) - answer_pool = maxpool(answer_dropout) + question_pool = maxpool(question_cnn) + answer_pool = maxpool(answer_cnn) - # activation - larger_dropout = Dropout(0.5) - activation = Activation('linear') - question_output = larger_dropout(activation(question_pool)) - answer_output = larger_dropout(activation(answer_pool)) - - return question_output, answer_output + return question_pool, answer_pool class AttentionModel(LanguageModel): @@ -283,10 +263,4 @@ class AttentionModel(LanguageModel): answer_b_rnn = b_rnn(answer_embedding) answer_pool = merge([maxpool(answer_f_rnn), maxpool(answer_b_rnn)], mode='concat', concat_axis=-1) - # activation - dropout = Dropout(0.5) - activation = Activation('linear') - question_output = activation(dropout(question_pool)) - answer_output = activation(dropout(answer_pool)) - - return question_output, answer_output + return question_pool, answer_pool diff --git a/seq2seq/answer_to_question.py b/seq2seq/answer_to_question.py index b15262c..13307c4 100644 --- a/seq2seq/answer_to_question.py +++ b/seq2seq/answer_to_question.py @@ -127,7 +127,6 @@ class EmbeddingRNNModel(LanguageModel): output_dim=self.model_params.get('n_embed_dims', 100), # W_regularizer=regularizers.activity_l1(1e-4), W_constraint=constraints.nonneg(), - dropout=0.5, weights=weights, mask_zero=True) question_embedding = embedding(question)