From 17b0797d7f2d159cadbd999a85209a8b6da2e914 Mon Sep 17 00:00:00 2001 From: codekansas Date: Sun, 31 Jul 2016 13:18:08 -0700 Subject: [PATCH] not sure what i changed --- insurance_qa_eval.py | 23 +++++-------- keras_models.py | 64 ++++++++++------------------------- seq2seq/answer_to_question.py | 1 - 3 files changed, 26 insertions(+), 62 deletions(-) diff --git a/insurance_qa_eval.py b/insurance_qa_eval.py index 5a568c8..61010bf 100644 --- a/insurance_qa_eval.py +++ b/insurance_qa_eval.py @@ -26,7 +26,7 @@ class Evaluator: self.path = data_path self.conf = dict() if conf is None else conf self.params = conf.get('training_params', dict()) - self.answers = self.load('answers') # self.load('generated') + self.answers = self.load('answers') # self.load('generated') self._vocab = None self._reverse_vocab = None self._eval_sets = None @@ -88,7 +88,6 @@ class Evaluator: print(strftime('%Y-%m-%d %H:%M:%S :: ', gmtime()), end='') def train(self, model): - eval_every = self.params.get('eval_every', None) save_every = self.params.get('save_every', None) batch_size = self.params.get('batch_size', 128) nb_epoch = self.params.get('nb_epoch', 10) @@ -112,11 +111,6 @@ class Evaluator: # sample from all answers to get bad answers bad_answers = self.pada(random.sample(self.answers.values(), len(good_answers))) - # shuffle questions - zipped = zip(questions, good_answers) - random.shuffle(zipped) - questions[:], good_answers[:] = zip(*zipped) - print('Epoch %d :: ' % i, end='') self.print_time() hist = model.fit([questions, good_answers, bad_answers], nb_epoch=1, batch_size=batch_size, @@ -177,7 +171,7 @@ class Evaluator: max_r = np.argmax(r) max_n = np.argmax(r[:n_good]) - # print(' '.join(self.revert(d['question']))) + # print(' '.join(self.revegrt(d['question']))) # print(' '.join(self.revert(self.answers[indices[max_r]]))) # print(' '.join(self.revert(self.answers[indices[max_n]]))) @@ -222,8 +216,8 @@ if __name__ == '__main__': import numpy as np conf = { - 'question_len': 200, - 'answer_len': 200, + 'question_len': 50, + 'answer_len': 100, 'n_words': 22353, # len(vocabulary) + 1 'margin': 0.2, @@ -247,10 +241,11 @@ if __name__ == '__main__': 'n_lstm_dims': 141, # * 2 'initial_embed_weights': np.load('models/word2vec_100_dim.h5'), + 'similarity_dropout': 0.2, }, 'similarity_params': { - 'mode': 'gesd', + 'mode': 'cosine', 'gamma': 1, 'c': 1, 'd': 2, @@ -260,7 +255,7 @@ if __name__ == '__main__': evaluator = Evaluator(conf) ##### Define model ###### - model = AttentionModel(conf) + model = EmbeddingModel(conf) optimizer = conf.get('training_params', dict()).get('optimizer', 'adam') model.compile(optimizer=optimizer) @@ -271,12 +266,12 @@ if __name__ == '__main__': # np.save(open('models/embedding_1000_dim.h5', 'wb'), weights) # train the model - # evaluator.load_epoch(model, 42) + # evaluator.load_epoch(model, 6) best_loss = evaluator.train(model) # evaluate mrr for a particular epoch evaluator.load_epoch(model, best_loss['epoch']) - # evaluator.load_epoch(model, 31) + # evaluator.load_epoch(model, 68) evaluator.get_mrr(model, evaluate_all=True) # for epoch in range(1, 100): # print('Epoch %d' % epoch) diff --git a/keras_models.py b/keras_models.py index 07d44cf..219d07c 100644 --- a/keras_models.py +++ b/keras_models.py @@ -105,31 +105,32 @@ class LanguageModel: question_output, answer_output = self._models similarity = self.get_similarity() - qa_model = merge([question_output, answer_output], mode=similarity, output_shape=lambda x: x[:-1]) + qa_model = merge([question_output, answer_output], mode=similarity, output_shape=lambda _: (None, 1)) + dropout = Dropout(self.similarity_params.get('similarity_dropout', 0.2))(qa_model) - self._qa_model = Model(input=[self.question, self.get_answer()], output=[qa_model]) + self._qa_model = Model(input=[self.question, self.get_answer()], output=[dropout]) return self._qa_model def compile(self, optimizer, **kwargs): qa_model = self.get_qa_model() - good_output = qa_model([self.question, self.answer_good]) - bad_output = qa_model([self.question, self.answer_bad]) + good_similarity = qa_model([self.question, self.answer_good]) + bad_similarity = qa_model([self.question, self.answer_bad]) - loss = merge([good_output, bad_output], + loss = merge([good_similarity, bad_similarity], mode=lambda x: K.relu(self.config['margin'] - x[0] + x[1]), output_shape=lambda x: x[0]) self.training_model = Model(input=[self.question, self.answer_good, self.answer_bad], output=loss) self.training_model.compile(loss=lambda y_true, y_pred: y_pred, optimizer=optimizer, **kwargs) - self.prediction_model = Model(input=[self.question, self.answer_good], output=good_output) + self.prediction_model = Model(input=[self.question, self.answer_good], output=good_similarity) self.prediction_model.compile(loss='binary_crossentropy', optimizer=optimizer, **kwargs) def fit(self, x, **kwargs): assert self.training_model is not None, 'Must compile the model before fitting data' - y = np.zeros(shape=x[0].shape[:1]) + y = np.zeros(shape=(x[0].shape[0],)) return self.training_model.fit(x, y, **kwargs) def predict(self, x, **kwargs): @@ -160,22 +161,12 @@ class EmbeddingModel(LanguageModel): question_embedding = embedding(question) answer_embedding = embedding(answer) - # dropout - dropout = Dropout(0.5) - question_dropout = dropout(question_embedding) - answer_dropout = dropout(answer_embedding) - # maxpooling maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) - question_maxpool = maxpool(question_dropout) - answer_maxpool = maxpool(answer_dropout) + question_pool = maxpool(question_embedding) + answer_pool = maxpool(answer_embedding) - # activation - activation = Activation('linear') - question_output = activation(question_maxpool) - answer_output = activation(answer_maxpool) - - return question_output, answer_output + return question_pool, answer_pool class ConvolutionModel(LanguageModel): @@ -200,18 +191,13 @@ class ConvolutionModel(LanguageModel): # embedding.params = [] # embedding.updates = [] - # dropout - dropout = Dropout(0.5) - question_dropout = dropout(question_embedding) - answer_dropout = dropout(answer_embedding) - # dense dense = TimeDistributed(Dense(self.model_params.get('n_hidden', 200), # activity_regularizer=regularizers.activity_l1(1e-4), # W_regularizer=regularizers.l1(1e-4), activation='tanh')) - question_dense = dropout(dense(question_dropout)) - answer_dense = dropout(dense(answer_dropout)) + question_dense = dense(question_embedding) + answer_dense = dense(answer_embedding) # cnn cnns = [Convolution1D(filter_length=filter_length, @@ -223,23 +209,13 @@ class ConvolutionModel(LanguageModel): question_cnn = merge([cnn(question_dense) for cnn in cnns], mode='concat') answer_cnn = merge([cnn(answer_dense) for cnn in cnns], mode='concat') - # dropout - question_dropout = dropout(question_cnn) - answer_dropout = dropout(answer_cnn) - # maxpooling maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) avepool = Lambda(lambda x: K.mean(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) - question_pool = maxpool(question_dropout) - answer_pool = maxpool(answer_dropout) + question_pool = maxpool(question_cnn) + answer_pool = maxpool(answer_cnn) - # activation - larger_dropout = Dropout(0.5) - activation = Activation('linear') - question_output = larger_dropout(activation(question_pool)) - answer_output = larger_dropout(activation(answer_pool)) - - return question_output, answer_output + return question_pool, answer_pool class AttentionModel(LanguageModel): @@ -283,10 +259,4 @@ class AttentionModel(LanguageModel): answer_b_rnn = b_rnn(answer_embedding) answer_pool = merge([maxpool(answer_f_rnn), maxpool(answer_b_rnn)], mode='concat', concat_axis=-1) - # activation - dropout = Dropout(0.5) - activation = Activation('linear') - question_output = activation(dropout(question_pool)) - answer_output = activation(dropout(answer_pool)) - - return question_output, answer_output + return question_pool, answer_pool diff --git a/seq2seq/answer_to_question.py b/seq2seq/answer_to_question.py index b15262c..13307c4 100644 --- a/seq2seq/answer_to_question.py +++ b/seq2seq/answer_to_question.py @@ -127,7 +127,6 @@ class EmbeddingRNNModel(LanguageModel): output_dim=self.model_params.get('n_embed_dims', 100), # W_regularizer=regularizers.activity_l1(1e-4), W_constraint=constraints.nonneg(), - dropout=0.5, weights=weights, mask_zero=True) question_embedding = embedding(question)