added script for generating embeddings, changed models a bit

generate_insurance_qa_embeddings.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import os
+import sys
+import random
+import pickle
+import argparse
+import logging
+
+random.seed(42)
+
+
+def load(path, name):
+    return pickle.load(open(os.path.join(path, name), 'rb'))
+
+
+def revert(vocab, indices):
+    return [vocab.get(i, 'X') for i in indices]
+
+try:
+    data_path = os.environ['INSURANCE_QA']
+except KeyError:
+    print('INSURANCE_QA is not set. Set it to your clone of https://github.com/codekansas/insurance_qa_python')
+    sys.exit(1)
+
+# parse arguments
+parser = argparse.ArgumentParser(description='Generate embeddings for the InsuranceQA dataset')
+parser.add_argument('--iter', metavar='N', type=int, nargs=1, default=10, help='number of times to run')
+parser.add_argument('--size', metavar='D', type=int, nargs=1, default=100, help='dimensions in embedding')
+args = parser.parse_args()
+
+# configure logging
+logger = logging.getLogger(os.path.basename(sys.argv[0]))
+logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s')
+logging.root.setLevel(level=logging.INFO)
+logger.info('running %s' % ' '.join(sys.argv))
+
+# imports go down here because they are time-consuming
+from gensim.models import Word2Vec
+from keras_models import *
+
+vocab = load(data_path, 'vocabulary')
+
+answers = load(data_path, 'answers')
+sentences = [revert(vocab, txt) for txt in answers.values()]
+sentences += [revert(vocab, q['question']) for q in load(data_path, 'train')]
+
+# run model
+model = Word2Vec(sentences, size=args.size, min_count=5, window=5, sg=1, iter=args.iter)
+weights = model.syn0
+d = dict([(k, v.index) for k, v in model.vocab.items()])
+emb = np.zeros(shape=(len(vocab)+1, args.size), dtype='float32')
+
+for i, w in vocab.items():
+    if w not in d: continue
+    emb[i, :] = weights[d[w], :]
+
+np.save(open('word2vec_%d_dim.embeddings' % args.size, 'wb'), emb)
+logger.info('saved to "word2vec_%d_dim.embeddings"' % args.size)
+

install.sh

@@ -4,7 +4,7 @@
 # Ubuntu 16.04 (as many CPUs as you like)
 
 # exit on failure
-set -e
+# set -e
 
 # make models directory
 if [ ! -d "models/" ]; then
@@ -41,6 +41,9 @@ fi
 cd $KERAS_DIRECTORY
 python setup.py install
 cd -
+if [ ! -d ~/.keras ]; then
+  mkdir ~/.keras
+fi
 echo '{"epsilon": 1e-07, "floatx": "float32", "backend": "tensorflow"}' > ~/.keras/keras.json
 
 # download insurance qa files

insurance_qa_eval.py

@@ -131,7 +131,7 @@ class Evaluator:
 
             print('Fitting epoch %d' % i, file=sys.stderr)
             hist = self.model.fit([questions, good_answers, bad_answers], nb_epoch=1, batch_size=batch_size,
-                             validation_split=validation_split, verbose=0)
+                             validation_split=validation_split, verbose=1)
 
             if hist.history['val_loss'][0] < val_loss['loss']:
                 val_loss = {'loss': hist.history['val_loss'][0], 'epoch': i}
@@ -231,7 +231,7 @@ if __name__ == '__main__':
 
     conf = {
         'n_words': 22353,
-        'question_len': 150,
+        'question_len': 20,
         'answer_len': 150,
         'margin': 0.05,
         'initial_embed_weights': 'word2vec_100_dim.embeddings',
@@ -243,15 +243,16 @@ if __name__ == '__main__':
         },
 
         'similarity': {
-            'mode': 'cosine',
+            'mode': 'gesd',
             'gamma': 1,
             'c': 1,
             'd': 2,
+            'dropout': 0.5,
         }
     }
 
     from keras_models import EmbeddingModel
-    evaluator = Evaluator(conf, model=EmbeddingModel, optimizer='sgd')
+    evaluator = Evaluator(conf, model=EmbeddingModel, optimizer='adam')
 
     # train the model
     best_loss = evaluator.train()

keras_models.py

@@ -97,7 +97,7 @@ class LanguageModel:
 
         if self._qa_model is None:
             question_output, answer_output = self._models
-            dropout = Dropout(self.params.get('similarity_dropout', 0.2))
+            dropout = Dropout(self.params.get('dropout', 0.2))
             similarity = self.get_similarity()
             qa_model = merge([dropout(question_output), dropout(answer_output)],
                              mode=similarity, output_shape=lambda _: (None, 1))
@@ -148,6 +148,8 @@ class EmbeddingModel(LanguageModel):
         weights = np.load(self.config['initial_embed_weights'])
         embedding = Embedding(input_dim=self.config['n_words'],
                               output_dim=weights.shape[1],
+                              mask_zero=True,
+                              # dropout=0.2,
                               weights=[weights])
         question_embedding = embedding(question)
         answer_embedding = embedding(answer)