updated results.notes

insurance_qa_eval.py

@@ -201,7 +201,7 @@ if __name__ == '__main__':
     data_path = '/media/moloch/HHD/MachineLearning/data/insuranceQA/pyenc'
 
     conf = {
-        'question_len': 100,
+        'question_len': 20,
         'answer_len': 100,
         'n_words': 22353, # len(vocabulary) + 1
         'margin': 0.009,
@@ -234,7 +234,7 @@ if __name__ == '__main__':
         },
 
         'similarity_params': {
-            'mode': 'cosine',
+            'mode': 'gesd',
             'gamma': 1,
             'c': 1,
             'd': 2,
@@ -244,7 +244,7 @@ if __name__ == '__main__':
     evaluator = Evaluator(data_path, conf)
 
     ##### Define model ######
-    model = ConvolutionModel(conf)
+    model = AttentionModel(conf)
     optimizer = conf.get('training_params', dict()).get('optimizer', 'adam')
     model.compile(optimizer=optimizer)
 
@@ -263,9 +263,9 @@ if __name__ == '__main__':
     language_model.layers[2].set_weights([weights])
 
     # train the model
-    # evaluator.load_epoch(model, 40)
+    # evaluator.load_epoch(model, 25)
     # evaluator.train(model)
 
     # evaluate mrr for a particular epoch
-    evaluator.load_epoch(model, 4)
+    evaluator.load_epoch(model, 115)
     evaluator.get_mrr(model, evaluate_all=True)

keras_models.py

@@ -15,9 +15,9 @@ from attention_lstm import AttentionLSTM
 
 class LanguageModel:
     def __init__(self, config):
-        self.question = Input(shape=(config['question_len'],), dtype='int32', name='question')
-        self.answer_good = Input(shape=(config['answer_len'],), dtype='int32', name='answer_good')
-        self.answer_bad = Input(shape=(config['answer_len'],), dtype='int32', name='answer_bad')
+        self.question = Input(shape=(config['question_len'],), dtype='int32', name='question_base')
+        self.answer_good = Input(shape=(config['answer_len'],), dtype='int32', name='answer_good_base')
+        self.answer_bad = Input(shape=(config['answer_len'],), dtype='int32', name='answer_bad_base')
 
         self.config = config
         self.model_params = config.get('model_params', dict())
@@ -26,17 +26,16 @@ class LanguageModel:
         # initialize a bunch of variables that will be set later
         self._models = None
         self._similarities = None
-        self._inputs = None
+        self._answer = None
         self._qa_model = None
 
         self.training_model = None
         self.prediction_model = None
 
-    def _get_inputs(self):
-        if self._inputs is None:
-            self._inputs = [Input(shape=(self.config['question_len'],), dtype='int32', name='question'),
-                            Input(shape=(self.config['answer_len'],), dtype='int32', name='answer')]
-        return self._inputs
+    def get_answer(self):
+        if self._answer is None:
+            self._answer = Input(shape=(self.config['answer_len'],), dtype='int32', name='answer')
+        return self._answer
 
     @abstractmethod
     def build(self):
@@ -107,7 +106,7 @@ class LanguageModel:
             similarity = self.get_similarity()
             qa_model = merge([question_output, answer_output], mode=similarity, output_shape=lambda x: x[:-1])
 
-            self._qa_model = Model(input=self._get_inputs(), output=[qa_model])
+            self._qa_model = Model(input=[self.question, self.get_answer()], output=[qa_model])
 
         return self._qa_model
 
@@ -146,7 +145,8 @@ class LanguageModel:
 
 class EmbeddingModel(LanguageModel):
     def build(self):
-        question, answer = self._get_inputs()
+        question = self.question
+        answer = self.get_answer()
 
         # add embedding layers
         embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 141))
@@ -175,7 +175,8 @@ class ConvolutionModel(LanguageModel):
     def build(self):
         assert self.config['question_len'] == self.config['answer_len']
 
-        question, answer = self._get_inputs()
+        question = self.question
+        answer = self.get_answer()
 
         # add embedding layers
         embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 100))
@@ -235,7 +236,8 @@ class ConvolutionModel(LanguageModel):
 
 class AttentionModel(LanguageModel):
     def build(self):
-        question, answer = self._get_inputs()
+        question = self.question
+        answer = self.get_answer()
 
         # add embedding layers
         embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 100))
@@ -266,6 +268,8 @@ class AttentionModel(LanguageModel):
         # answer rnn part
         f_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, return_sequences=True)
         b_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, return_sequences=True, go_backwards=True)
+        # f_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True)
+        # b_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True, go_backwards=True)
         answer_rnn = merge([f_rnn(answer_dropout), b_rnn(answer_dropout)], mode='concat', concat_axis=-1)
         answer_dropout = dropout(answer_rnn)
         answer_pool = maxpool(answer_dropout)

results.notes

@@ -1,91 +1,16 @@
-Single-layer bi-LSTM with max pooling, 40 words per sentence, loss margin of 0.2
- - MRR ~0.17
- - Seemed to converge after about 20 epochs, with randomization between epochs
- - Using the pure embedding layer worked better than using the Word2Vec model (gave MRR ~0.09)
+Best results achieved for each model:
 
-CNN + ALSTM model seems to have good loss
- - In general, the model predicts the topics well, but doesn't necessarily match it well with the question. This might be due to the masking issue.
- - MRR: 0.30957
-
-2 ALSTM + MaxPooling
- - MRR: 0.0147 (didn't converge)
-
-Pure CNN Model:
- - 4000 filters
- - Maxpooling
- - After 120 epochs, InsuranceQA MRR = 0.530 and Top-1 Precision = 0.401
-   - Decreased slightly with further training due to overfitting
- - Mixing together different filter lengths improved it to MRR = 0.56 and Top-1 Precision = 0.44
- - Validation loss around 6e-4 (lower than maxpooling) underperformed compared to maxpooling
-
-Embedding + MaxPooling:
- - I can't believe this model performed so well. It blew the other ones out of the water, and trains ridiculously quickly.
- - Test 1: Top-1 Precision = 0.4922, MRR = 0.6239
- - Test 2: Top-1 Precision = 0.4817, MRR = 0.6110
- - Dev: Top-1 Precision = 0.4950, MRR = 0.6244
- - With 100 dimensions, Top-1 Precision = 0.281, MRR = 0.417 on test 1
- - Adding more embedding dimensions (beyond 1000) didn't lead to an improvement
- - Converted after about 20 epochs
- - Validation loss was around 7e-4 (with margin of 0.009)
- - Configuration:
-    conf = {
-        'question_len': 100,
-        'answer_len': 100,
-        'n_words': 22353, # len(vocabulary) + 1
-        'margin': 0.009,
-
-        'training_params': {
-            'save_every': 1,
-            'eval_every': 20,
-            'batch_size': 128,
-            'nb_epoch': 1000,
-            'validation_split': 0.2,
-            'optimizer': 'adam',
-            # 'n_eval': 20,
-        },
-
-        'model_params': {
-            'n_embed_dims': 1000,
-            'n_hidden': 200,
-
-            # convolution
-            'nb_filters': 1000,
-            'conv_activation': 'relu',
-
-            # recurrent
-            'n_lstm_dims': 300,
-        },
-
-        'similarity_params': {
-            'mode': 'cosine',
-            'gamma': 1,
-            'c': 1,
-            'd': 2,
-        }
-    }
-
-
-Model described in paper (Dense + CNN):
- - Top 1 precision:
-   - 0.183 on test 1
-   - 0.178 on test 2
-   - 0.204 on dev
+Embedding + Max Pooling:
+ - Top 1 Precision:
+   - 0.492 on test 1
+   - 0.483 on test 2
+   - 0.495 on dev
  - MRR:
-   - 0.328 on test 1
-   - 0.319 on test 2
-   - 0.343 on dev
- - Why won't this train better :(
+   - 0.624 on test 1
+   - 0.611 on test 2
+   - 0.624 on dev
 
- - Best validation loss was about 0.0011 (for margin of 0.009)
- - After increasing the number of parameters/embedding size the model improved a bit
- - Top 1 precision:
-   - 0.327 on test 1
-   - 0.302 on test 2
- - MRR:
-   - 0.459 on test 1
-   - 0.433 on test 2
-
- - Pre-trained the embedding layer via the EmbeddingModel, re-ran
+Dense + CNN + Max Pooling:
  - Top 1 precision:
    - 0.507 on test 1
    - 0.427 on test 2
@@ -95,8 +20,13 @@ Model described in paper (Dense + CNN):
    - 0.557 on test 2
    - 0.596 on dev
 
-1000 embed dims + 2000 CNN filters (total over 4 lengths):
- - Loss ~6e-4 for margin of 0.009
- - Test 1: 0.409 Top-1 Precision, 0.543 MRR
- - Test 2: 0.376 Top-1 Precision, 0.507 MRR
+Attentional LSTM + Max Pooling:
+ - Top 1 precision:
+   - 0.480 on test 1
+   - 0.465 on test 2
+   - 0.487 on dev
+ - MRR:
+   - 0.627 on test 1
+   - 0.613 on test 2
+   - 0.635 on dev