From 885eb7ebad2dadcc25fbcfd341f4a749ec7b5f7d Mon Sep 17 00:00:00 2001 From: codekansas Date: Tue, 3 May 2016 12:28:12 -0400 Subject: [PATCH] updated results.notes --- insurance_qa_eval.py | 10 ++-- keras_models.py | 30 ++++++------ results.notes | 108 ++++++++----------------------------------- 3 files changed, 41 insertions(+), 107 deletions(-) diff --git a/insurance_qa_eval.py b/insurance_qa_eval.py index 52aabed..cc399c4 100644 --- a/insurance_qa_eval.py +++ b/insurance_qa_eval.py @@ -201,7 +201,7 @@ if __name__ == '__main__': data_path = '/media/moloch/HHD/MachineLearning/data/insuranceQA/pyenc' conf = { - 'question_len': 100, + 'question_len': 20, 'answer_len': 100, 'n_words': 22353, # len(vocabulary) + 1 'margin': 0.009, @@ -234,7 +234,7 @@ if __name__ == '__main__': }, 'similarity_params': { - 'mode': 'cosine', + 'mode': 'gesd', 'gamma': 1, 'c': 1, 'd': 2, @@ -244,7 +244,7 @@ if __name__ == '__main__': evaluator = Evaluator(data_path, conf) ##### Define model ###### - model = ConvolutionModel(conf) + model = AttentionModel(conf) optimizer = conf.get('training_params', dict()).get('optimizer', 'adam') model.compile(optimizer=optimizer) @@ -263,9 +263,9 @@ if __name__ == '__main__': language_model.layers[2].set_weights([weights]) # train the model - # evaluator.load_epoch(model, 40) + # evaluator.load_epoch(model, 25) # evaluator.train(model) # evaluate mrr for a particular epoch - evaluator.load_epoch(model, 4) + evaluator.load_epoch(model, 115) evaluator.get_mrr(model, evaluate_all=True) diff --git a/keras_models.py b/keras_models.py index 7197079..97b9eba 100644 --- a/keras_models.py +++ b/keras_models.py @@ -15,9 +15,9 @@ from attention_lstm import AttentionLSTM class LanguageModel: def __init__(self, config): - self.question = Input(shape=(config['question_len'],), dtype='int32', name='question') - self.answer_good = Input(shape=(config['answer_len'],), dtype='int32', name='answer_good') - self.answer_bad = Input(shape=(config['answer_len'],), dtype='int32', name='answer_bad') + self.question = Input(shape=(config['question_len'],), dtype='int32', name='question_base') + self.answer_good = Input(shape=(config['answer_len'],), dtype='int32', name='answer_good_base') + self.answer_bad = Input(shape=(config['answer_len'],), dtype='int32', name='answer_bad_base') self.config = config self.model_params = config.get('model_params', dict()) @@ -26,17 +26,16 @@ class LanguageModel: # initialize a bunch of variables that will be set later self._models = None self._similarities = None - self._inputs = None + self._answer = None self._qa_model = None self.training_model = None self.prediction_model = None - def _get_inputs(self): - if self._inputs is None: - self._inputs = [Input(shape=(self.config['question_len'],), dtype='int32', name='question'), - Input(shape=(self.config['answer_len'],), dtype='int32', name='answer')] - return self._inputs + def get_answer(self): + if self._answer is None: + self._answer = Input(shape=(self.config['answer_len'],), dtype='int32', name='answer') + return self._answer @abstractmethod def build(self): @@ -107,7 +106,7 @@ class LanguageModel: similarity = self.get_similarity() qa_model = merge([question_output, answer_output], mode=similarity, output_shape=lambda x: x[:-1]) - self._qa_model = Model(input=self._get_inputs(), output=[qa_model]) + self._qa_model = Model(input=[self.question, self.get_answer()], output=[qa_model]) return self._qa_model @@ -146,7 +145,8 @@ class LanguageModel: class EmbeddingModel(LanguageModel): def build(self): - question, answer = self._get_inputs() + question = self.question + answer = self.get_answer() # add embedding layers embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 141)) @@ -175,7 +175,8 @@ class ConvolutionModel(LanguageModel): def build(self): assert self.config['question_len'] == self.config['answer_len'] - question, answer = self._get_inputs() + question = self.question + answer = self.get_answer() # add embedding layers embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 100)) @@ -235,7 +236,8 @@ class ConvolutionModel(LanguageModel): class AttentionModel(LanguageModel): def build(self): - question, answer = self._get_inputs() + question = self.question + answer = self.get_answer() # add embedding layers embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 100)) @@ -266,6 +268,8 @@ class AttentionModel(LanguageModel): # answer rnn part f_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, return_sequences=True) b_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, return_sequences=True, go_backwards=True) + # f_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True) + # b_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True, go_backwards=True) answer_rnn = merge([f_rnn(answer_dropout), b_rnn(answer_dropout)], mode='concat', concat_axis=-1) answer_dropout = dropout(answer_rnn) answer_pool = maxpool(answer_dropout) diff --git a/results.notes b/results.notes index df1caa2..a68f2d2 100644 --- a/results.notes +++ b/results.notes @@ -1,91 +1,16 @@ -Single-layer bi-LSTM with max pooling, 40 words per sentence, loss margin of 0.2 - - MRR ~0.17 - - Seemed to converge after about 20 epochs, with randomization between epochs - - Using the pure embedding layer worked better than using the Word2Vec model (gave MRR ~0.09) +Best results achieved for each model: -CNN + ALSTM model seems to have good loss - - In general, the model predicts the topics well, but doesn't necessarily match it well with the question. This might be due to the masking issue. - - MRR: 0.30957 - -2 ALSTM + MaxPooling - - MRR: 0.0147 (didn't converge) - -Pure CNN Model: - - 4000 filters - - Maxpooling - - After 120 epochs, InsuranceQA MRR = 0.530 and Top-1 Precision = 0.401 - - Decreased slightly with further training due to overfitting - - Mixing together different filter lengths improved it to MRR = 0.56 and Top-1 Precision = 0.44 - - Validation loss around 6e-4 (lower than maxpooling) underperformed compared to maxpooling - -Embedding + MaxPooling: - - I can't believe this model performed so well. It blew the other ones out of the water, and trains ridiculously quickly. - - Test 1: Top-1 Precision = 0.4922, MRR = 0.6239 - - Test 2: Top-1 Precision = 0.4817, MRR = 0.6110 - - Dev: Top-1 Precision = 0.4950, MRR = 0.6244 - - With 100 dimensions, Top-1 Precision = 0.281, MRR = 0.417 on test 1 - - Adding more embedding dimensions (beyond 1000) didn't lead to an improvement - - Converted after about 20 epochs - - Validation loss was around 7e-4 (with margin of 0.009) - - Configuration: - conf = { - 'question_len': 100, - 'answer_len': 100, - 'n_words': 22353, # len(vocabulary) + 1 - 'margin': 0.009, - - 'training_params': { - 'save_every': 1, - 'eval_every': 20, - 'batch_size': 128, - 'nb_epoch': 1000, - 'validation_split': 0.2, - 'optimizer': 'adam', - # 'n_eval': 20, - }, - - 'model_params': { - 'n_embed_dims': 1000, - 'n_hidden': 200, - - # convolution - 'nb_filters': 1000, - 'conv_activation': 'relu', - - # recurrent - 'n_lstm_dims': 300, - }, - - 'similarity_params': { - 'mode': 'cosine', - 'gamma': 1, - 'c': 1, - 'd': 2, - } - } - - -Model described in paper (Dense + CNN): - - Top 1 precision: - - 0.183 on test 1 - - 0.178 on test 2 - - 0.204 on dev +Embedding + Max Pooling: + - Top 1 Precision: + - 0.492 on test 1 + - 0.483 on test 2 + - 0.495 on dev - MRR: - - 0.328 on test 1 - - 0.319 on test 2 - - 0.343 on dev - - Why won't this train better :( + - 0.624 on test 1 + - 0.611 on test 2 + - 0.624 on dev - - Best validation loss was about 0.0011 (for margin of 0.009) - - After increasing the number of parameters/embedding size the model improved a bit - - Top 1 precision: - - 0.327 on test 1 - - 0.302 on test 2 - - MRR: - - 0.459 on test 1 - - 0.433 on test 2 - - - Pre-trained the embedding layer via the EmbeddingModel, re-ran +Dense + CNN + Max Pooling: - Top 1 precision: - 0.507 on test 1 - 0.427 on test 2 @@ -95,8 +20,13 @@ Model described in paper (Dense + CNN): - 0.557 on test 2 - 0.596 on dev -1000 embed dims + 2000 CNN filters (total over 4 lengths): - - Loss ~6e-4 for margin of 0.009 - - Test 1: 0.409 Top-1 Precision, 0.543 MRR - - Test 2: 0.376 Top-1 Precision, 0.507 MRR +Attentional LSTM + Max Pooling: + - Top 1 precision: + - 0.480 on test 1 + - 0.465 on test 2 + - 0.487 on dev + - MRR: + - 0.627 on test 1 + - 0.613 on test 2 + - 0.635 on dev