diff --git a/generate_insurance_qa_embeddings.py b/generate_insurance_qa_embeddings.py new file mode 100755 index 0000000..68097c7 --- /dev/null +++ b/generate_insurance_qa_embeddings.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import os +import sys +import random +import pickle +import argparse +import logging + +random.seed(42) + + +def load(path, name): + return pickle.load(open(os.path.join(path, name), 'rb')) + + +def revert(vocab, indices): + return [vocab.get(i, 'X') for i in indices] + +try: + data_path = os.environ['INSURANCE_QA'] +except KeyError: + print('INSURANCE_QA is not set. Set it to your clone of https://github.com/codekansas/insurance_qa_python') + sys.exit(1) + +# parse arguments +parser = argparse.ArgumentParser(description='Generate embeddings for the InsuranceQA dataset') +parser.add_argument('--iter', metavar='N', type=int, nargs=1, default=10, help='number of times to run') +parser.add_argument('--size', metavar='D', type=int, nargs=1, default=100, help='dimensions in embedding') +args = parser.parse_args() + +# configure logging +logger = logging.getLogger(os.path.basename(sys.argv[0])) +logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s') +logging.root.setLevel(level=logging.INFO) +logger.info('running %s' % ' '.join(sys.argv)) + +# imports go down here because they are time-consuming +from gensim.models import Word2Vec +from keras_models import * + +vocab = load(data_path, 'vocabulary') + +answers = load(data_path, 'answers') +sentences = [revert(vocab, txt) for txt in answers.values()] +sentences += [revert(vocab, q['question']) for q in load(data_path, 'train')] + +# run model +model = Word2Vec(sentences, size=args.size, min_count=5, window=5, sg=1, iter=args.iter) +weights = model.syn0 +d = dict([(k, v.index) for k, v in model.vocab.items()]) +emb = np.zeros(shape=(len(vocab)+1, args.size), dtype='float32') + +for i, w in vocab.items(): + if w not in d: continue + emb[i, :] = weights[d[w], :] + +np.save(open('word2vec_%d_dim.embeddings' % args.size, 'wb'), emb) +logger.info('saved to "word2vec_%d_dim.embeddings"' % args.size) + diff --git a/install.sh b/install.sh index b243de9..f2b7ac5 100755 --- a/install.sh +++ b/install.sh @@ -4,7 +4,7 @@ # Ubuntu 16.04 (as many CPUs as you like) # exit on failure -set -e +# set -e # make models directory if [ ! -d "models/" ]; then @@ -41,6 +41,9 @@ fi cd $KERAS_DIRECTORY python setup.py install cd - +if [ ! -d ~/.keras ]; then + mkdir ~/.keras +fi echo '{"epsilon": 1e-07, "floatx": "float32", "backend": "tensorflow"}' > ~/.keras/keras.json # download insurance qa files diff --git a/insurance_qa_eval.py b/insurance_qa_eval.py index 83bac7c..55e9b0d 100644 --- a/insurance_qa_eval.py +++ b/insurance_qa_eval.py @@ -131,7 +131,7 @@ class Evaluator: print('Fitting epoch %d' % i, file=sys.stderr) hist = self.model.fit([questions, good_answers, bad_answers], nb_epoch=1, batch_size=batch_size, - validation_split=validation_split, verbose=0) + validation_split=validation_split, verbose=1) if hist.history['val_loss'][0] < val_loss['loss']: val_loss = {'loss': hist.history['val_loss'][0], 'epoch': i} @@ -231,7 +231,7 @@ if __name__ == '__main__': conf = { 'n_words': 22353, - 'question_len': 150, + 'question_len': 20, 'answer_len': 150, 'margin': 0.05, 'initial_embed_weights': 'word2vec_100_dim.embeddings', @@ -243,15 +243,16 @@ if __name__ == '__main__': }, 'similarity': { - 'mode': 'cosine', + 'mode': 'gesd', 'gamma': 1, 'c': 1, 'd': 2, + 'dropout': 0.5, } } from keras_models import EmbeddingModel - evaluator = Evaluator(conf, model=EmbeddingModel, optimizer='sgd') + evaluator = Evaluator(conf, model=EmbeddingModel, optimizer='adam') # train the model best_loss = evaluator.train() diff --git a/keras_models.py b/keras_models.py index 134140b..85520ac 100644 --- a/keras_models.py +++ b/keras_models.py @@ -97,7 +97,7 @@ class LanguageModel: if self._qa_model is None: question_output, answer_output = self._models - dropout = Dropout(self.params.get('similarity_dropout', 0.2)) + dropout = Dropout(self.params.get('dropout', 0.2)) similarity = self.get_similarity() qa_model = merge([dropout(question_output), dropout(answer_output)], mode=similarity, output_shape=lambda _: (None, 1)) @@ -148,6 +148,8 @@ class EmbeddingModel(LanguageModel): weights = np.load(self.config['initial_embed_weights']) embedding = Embedding(input_dim=self.config['n_words'], output_dim=weights.shape[1], + mask_zero=True, + # dropout=0.2, weights=[weights]) question_embedding = embedding(question) answer_embedding = embedding(answer)