wrote convolution model, added word2vec

insurance_qa_embeddings.py

@@ -0,0 +1,49 @@
+from __future__ import print_function
+
+import os
+import random
+from time import strftime, gmtime
+
+import pickle
+
+from gensim.models import Word2Vec
+
+from keras_models import *
+
+random.seed(42)
+
+
+def load(path, name):
+    return pickle.load(open(os.path.join(path, name), 'rb'))
+
+
+def revert(vocab, indices):
+    return [vocab.get(i, 'X') for i in indices]
+
+if __name__ == '__main__':
+    data_path = '/media/moloch/HHD/MachineLearning/data/insuranceQA/pyenc'
+    vocab = load(data_path, 'vocab')
+
+    sentences = list()
+    answers = load(data_path, 'answers')
+    for id, txt in answers.items():
+        sentences.append(revert(vocab, txt))
+    for q in load(data_path, 'train'):
+        sentences.append(revert(vocab, q['question']))
+
+    model = Word2Vec(sentences, size=100, min_count=5, window=5, sg=1, iter=25)
+    weights = model.syn0
+    d = dict([(k, v.index) for k, v in model.vocab.items()])
+
+    # this is the stored weights of an equivalent embedding layer
+    emb = np.load('models/embedding_100_dim.h5')
+
+    # load the vocabulary
+    vocab = pickle.load(open(os.path.join(data_path, 'vocabulary'), 'rb'))
+
+    # swap the word2vec weights with the embedded weights
+    for i, w in vocab.items():
+        if w not in d: continue
+        emb[i, :] = weights[d[w], :]
+
+    np.save(open('models/word2vec_100_dim.h5', 'wb'), emb)

insurance_qa_eval.py

@@ -13,8 +13,6 @@ from keras_models import *
 
 random.seed(42)
 
-data_path = '/media/moloch/HHD/MachineLearning/data/insuranceQA/pyenc'
-
 
 class Evaluator:
     def __init__(self, path, conf=None):
@@ -188,11 +186,11 @@ class Evaluator:
             top1_threshold = evaluate_all_threshold.get('top1', 1)
 
             if evaluate_mode == 'any':
-                evaluate_all = evaluate_all or any([x > top1_threshold for x in top1s])
-                evaluate_all = evaluate_all or any([x > mrr_theshold for x in mrrs])
+                evaluate_all = evaluate_all or any([x >= top1_threshold for x in top1s])
+                evaluate_all = evaluate_all or any([x >= mrr_theshold for x in mrrs])
             else:
-                evaluate_all = evaluate_all or all([x > top1_threshold for x in top1s])
-                evaluate_all = evaluate_all or all([x > mrr_theshold for x in mrrs])
+                evaluate_all = evaluate_all or all([x >= top1_threshold for x in top1s])
+                evaluate_all = evaluate_all and all([x >= mrr_theshold for x in mrrs])
 
             if evaluate_all:
                 return self.get_mrr(model, evaluate_all=True)
@@ -200,6 +198,8 @@ class Evaluator:
         return top1s, mrrs
 
 if __name__ == '__main__':
+    data_path = '/media/moloch/HHD/MachineLearning/data/insuranceQA/pyenc'
+
     conf = {
         'question_len': 100,
         'answer_len': 100,
@@ -216,7 +216,7 @@ if __name__ == '__main__':
             'n_eval': 20,
 
             'evaluate_all_threshold': {
-                'mode': 'any',
+                'mode': 'all',
                 'top1': 0.55,
             },
         },
@@ -230,7 +230,7 @@ if __name__ == '__main__':
             'conv_activation': 'relu',
 
             # recurrent
-            'n_lstm_dims': 300,
+            'n_lstm_dims': 141,
         },
 
         'similarity_params': {
@@ -248,16 +248,24 @@ if __name__ == '__main__':
     optimizer = conf.get('training_params', dict()).get('optimizer', 'adam')
     model.compile(optimizer=optimizer)
 
-    # load pre-trained embedding layer
     import numpy as np
-    weights = np.load('models/embedding_100_dim.h5')
+
+    # save embedding layer
+    # embedding_layer = model.prediction_model.layers[2].layers[2]
+    # evaluator.load_epoch(model, 100)
+    # evaluator.train(model)
+    # weights = embedding_layer.get_weights()[0]
+    # np.save(open('models/embedding_200_dim.h5', 'wb'), weights)
+
+    # load pre-trained embedding layer
+    weights = np.load('models/word2vec_100_dim.h5')
     language_model = model.prediction_model.layers[2]
     language_model.layers[2].set_weights([weights])
 
     # train the model
-    evaluator.load_epoch(model, 10)
+    evaluator.load_epoch(model, 40)
     evaluator.train(model)
 
     # evaluate mrr for a particular epoch
-    # evaluator.load_epoch(model, 10)
-    # evaluator.get_mrr(model)
+    # evaluator.load_epoch(model, 52)
+    # evaluator.get_mrr(model, evaluate_all=True)

keras_models.py

@@ -172,14 +172,6 @@ class EmbeddingModel(LanguageModel):
 
 
 class ConvolutionModel(LanguageModel):
-    def mixed_filter_lengths(self, input, filter_lengths):
-        cnns = [Convolution1D(filter_length=filter_length,
-                              nb_filter=self.model_params.get('nb_filters', 1000),
-                              activation=self.model_params.get('conv_activation', 'relu'),
-                              border_mode='same') for filter_length in filter_lengths]
-
-        return merge([cnn(input) for cnn in cnns], mode='concat'), cnns
-
     def build(self):
         assert self.config['question_len'] == self.config['answer_len']
 
@@ -241,43 +233,46 @@ class ConvolutionModel(LanguageModel):
         return question_output, answer_output
 
 
-class RecurrentModel(LanguageModel):
+class AttentionModel(LanguageModel):
     def build(self):
-        input, _ = self._get_inputs()
+        question, answer = self._get_inputs()
 
         # add embedding layers
-        embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 141))
-        input_embedding = embedding(input)
+        embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 100))
+        question_embedding = embedding(question)
+        answer_embedding = embedding(answer)
+
+        # turn off layer updating
+        embedding.params = []
+        embedding.updates = []
 
         # dropout
-        dropout = Dropout(0.5)
-        input_dropout = dropout(input_embedding)
+        dropout = Dropout(0.25)
+        question_dropout = dropout(question_embedding)
+        answer_dropout = dropout(answer_embedding)
 
-        # rnn
-        forward_lstm = LSTM(self.config.get('n_lstm_dims', 141), consume_less='mem', return_sequences=True)
-        backward_lstm = LSTM(self.config.get('n_lstm_dims', 141), consume_less='mem', return_sequences=True)
-        input_lstm = merge([forward_lstm(input_dropout), backward_lstm(input_dropout)], mode='concat', concat_axis=-1)
+        # question rnn part
+        f_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True)
+        b_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True, go_backwards=True)
+        question_rnn = merge([f_rnn(question_dropout), b_rnn(question_dropout)], mode='concat', concat_axis=-1)
+        question_dropout = dropout(question_rnn)
 
-        # dropout
-        input_dropout = dropout(input_lstm)
-
-        # cnn
-        cnns = [Convolution1D(filter_length=filter_length,
-                              nb_filter=self.model_params.get('nb_filters', 1000),
-                              activation=self.model_params.get('conv_activation', 'relu'),
-                              border_mode='same') for filter_length in [2, 3, 5, 7]]
-        input_cnn = merge([cnn(input_dropout) for cnn in cnns], mode='concat')
-
-        # dropout
-        input_dropout = dropout(input_cnn)
+        # could add convolution layer here (as in paper)
 
         # maxpooling
-        maxpool = Lambda(lambda x: K.mean(K.exp(x), axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]))
-        input_pool = maxpool(input_dropout)
+        maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]))
+        question_pool = maxpool(question_dropout)
+
+        # answer rnn part
+        f_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, return_sequences=True)
+        b_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, return_sequences=True, go_backwards=True)
+        answer_rnn = merge([f_rnn(answer_dropout), b_rnn(answer_dropout)], mode='concat', concat_axis=-1)
+        answer_dropout = dropout(answer_rnn)
+        answer_pool = maxpool(answer_dropout)
 
         # activation
         activation = Activation('tanh')
-        output = activation(input_pool)
+        question_output = activation(question_pool)
+        answer_output = activation(answer_pool)
 
-        model = Model(input=[input], output=[output])
-        return model, model
+        return question_output, answer_output

results.notes

@@ -87,11 +87,11 @@ Model described in paper (Dense + CNN):
 
  - Pre-trained the embedding layer via the EmbeddingModel, re-ran
  - Top 1 precision:
-   - 0.472 on test 1
+   - 0.493 on test 1
    - 0.427 on test 2
    - 0.470 on dev
  - MRR:
-   - 0.595 on test 1
+   - 0.620 on test 1
    - 0.557 on test 2
    - 0.596 on dev
 

word_embeddings.py

@@ -1,28 +0,0 @@
-from gensim.models import Word2Vec
-from keras.engine import Layer
-
-import keras.backend as K
-
-
-class Word2VecEmbedding(Layer):
-    ''' This layer can be used instead of Keras's Embedding layer,
-    if word2vec encodings are desired. The performance is generally about
-    the same, although this isn't as nice of a way to do it. It uses gensim
-    to train the model, and takes the path to that model in the constructor.
-    '''
-    def __init__(self, model_path, **kwargs):
-        self.model = Word2Vec.load(model_path)
-        self.W = K.variable(self.model.syn0)
-        self.model_dims = self.model.syn0.shape
-        super(Word2VecEmbedding, self).__init__(**kwargs)
-
-    def build(self, input_shape):
-        self.trainable_weights = []
-
-    def get_output_shape_for(self, input_shape):
-        assert len(input_shape) == 2, 'Must provide a 2D input shape: (n_samples, input_vector)'
-        return (input_shape[0], input_shape[1], self.model_dims[1])
-
-    def call(self, x, mask=None):
-        x = K.maximum(K.minimum(x, self.model_dims[1] - 1), 0)
-        return K.gather(self.W, x)