diff --git a/insurance_qa_embeddings.py b/insurance_qa_embeddings.py new file mode 100644 index 0000000..cabd05b --- /dev/null +++ b/insurance_qa_embeddings.py @@ -0,0 +1,49 @@ +from __future__ import print_function + +import os +import random +from time import strftime, gmtime + +import pickle + +from gensim.models import Word2Vec + +from keras_models import * + +random.seed(42) + + +def load(path, name): + return pickle.load(open(os.path.join(path, name), 'rb')) + + +def revert(vocab, indices): + return [vocab.get(i, 'X') for i in indices] + +if __name__ == '__main__': + data_path = '/media/moloch/HHD/MachineLearning/data/insuranceQA/pyenc' + vocab = load(data_path, 'vocab') + + sentences = list() + answers = load(data_path, 'answers') + for id, txt in answers.items(): + sentences.append(revert(vocab, txt)) + for q in load(data_path, 'train'): + sentences.append(revert(vocab, q['question'])) + + model = Word2Vec(sentences, size=100, min_count=5, window=5, sg=1, iter=25) + weights = model.syn0 + d = dict([(k, v.index) for k, v in model.vocab.items()]) + + # this is the stored weights of an equivalent embedding layer + emb = np.load('models/embedding_100_dim.h5') + + # load the vocabulary + vocab = pickle.load(open(os.path.join(data_path, 'vocabulary'), 'rb')) + + # swap the word2vec weights with the embedded weights + for i, w in vocab.items(): + if w not in d: continue + emb[i, :] = weights[d[w], :] + + np.save(open('models/word2vec_100_dim.h5', 'wb'), emb) diff --git a/insurance_qa_eval.py b/insurance_qa_eval.py index 52dbc9b..5823dae 100644 --- a/insurance_qa_eval.py +++ b/insurance_qa_eval.py @@ -13,8 +13,6 @@ from keras_models import * random.seed(42) -data_path = '/media/moloch/HHD/MachineLearning/data/insuranceQA/pyenc' - class Evaluator: def __init__(self, path, conf=None): @@ -188,11 +186,11 @@ class Evaluator: top1_threshold = evaluate_all_threshold.get('top1', 1) if evaluate_mode == 'any': - evaluate_all = evaluate_all or any([x > top1_threshold for x in top1s]) - evaluate_all = evaluate_all or any([x > mrr_theshold for x in mrrs]) + evaluate_all = evaluate_all or any([x >= top1_threshold for x in top1s]) + evaluate_all = evaluate_all or any([x >= mrr_theshold for x in mrrs]) else: - evaluate_all = evaluate_all or all([x > top1_threshold for x in top1s]) - evaluate_all = evaluate_all or all([x > mrr_theshold for x in mrrs]) + evaluate_all = evaluate_all or all([x >= top1_threshold for x in top1s]) + evaluate_all = evaluate_all and all([x >= mrr_theshold for x in mrrs]) if evaluate_all: return self.get_mrr(model, evaluate_all=True) @@ -200,6 +198,8 @@ class Evaluator: return top1s, mrrs if __name__ == '__main__': + data_path = '/media/moloch/HHD/MachineLearning/data/insuranceQA/pyenc' + conf = { 'question_len': 100, 'answer_len': 100, @@ -216,7 +216,7 @@ if __name__ == '__main__': 'n_eval': 20, 'evaluate_all_threshold': { - 'mode': 'any', + 'mode': 'all', 'top1': 0.55, }, }, @@ -230,7 +230,7 @@ if __name__ == '__main__': 'conv_activation': 'relu', # recurrent - 'n_lstm_dims': 300, + 'n_lstm_dims': 141, }, 'similarity_params': { @@ -248,16 +248,24 @@ if __name__ == '__main__': optimizer = conf.get('training_params', dict()).get('optimizer', 'adam') model.compile(optimizer=optimizer) - # load pre-trained embedding layer import numpy as np - weights = np.load('models/embedding_100_dim.h5') + + # save embedding layer + # embedding_layer = model.prediction_model.layers[2].layers[2] + # evaluator.load_epoch(model, 100) + # evaluator.train(model) + # weights = embedding_layer.get_weights()[0] + # np.save(open('models/embedding_200_dim.h5', 'wb'), weights) + + # load pre-trained embedding layer + weights = np.load('models/word2vec_100_dim.h5') language_model = model.prediction_model.layers[2] language_model.layers[2].set_weights([weights]) # train the model - evaluator.load_epoch(model, 10) + evaluator.load_epoch(model, 40) evaluator.train(model) # evaluate mrr for a particular epoch - # evaluator.load_epoch(model, 10) - # evaluator.get_mrr(model) + # evaluator.load_epoch(model, 52) + # evaluator.get_mrr(model, evaluate_all=True) diff --git a/keras_models.py b/keras_models.py index 2c09326..7197079 100644 --- a/keras_models.py +++ b/keras_models.py @@ -172,14 +172,6 @@ class EmbeddingModel(LanguageModel): class ConvolutionModel(LanguageModel): - def mixed_filter_lengths(self, input, filter_lengths): - cnns = [Convolution1D(filter_length=filter_length, - nb_filter=self.model_params.get('nb_filters', 1000), - activation=self.model_params.get('conv_activation', 'relu'), - border_mode='same') for filter_length in filter_lengths] - - return merge([cnn(input) for cnn in cnns], mode='concat'), cnns - def build(self): assert self.config['question_len'] == self.config['answer_len'] @@ -241,43 +233,46 @@ class ConvolutionModel(LanguageModel): return question_output, answer_output -class RecurrentModel(LanguageModel): +class AttentionModel(LanguageModel): def build(self): - input, _ = self._get_inputs() + question, answer = self._get_inputs() # add embedding layers - embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 141)) - input_embedding = embedding(input) + embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 100)) + question_embedding = embedding(question) + answer_embedding = embedding(answer) + + # turn off layer updating + embedding.params = [] + embedding.updates = [] # dropout - dropout = Dropout(0.5) - input_dropout = dropout(input_embedding) + dropout = Dropout(0.25) + question_dropout = dropout(question_embedding) + answer_dropout = dropout(answer_embedding) - # rnn - forward_lstm = LSTM(self.config.get('n_lstm_dims', 141), consume_less='mem', return_sequences=True) - backward_lstm = LSTM(self.config.get('n_lstm_dims', 141), consume_less='mem', return_sequences=True) - input_lstm = merge([forward_lstm(input_dropout), backward_lstm(input_dropout)], mode='concat', concat_axis=-1) + # question rnn part + f_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True) + b_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True, go_backwards=True) + question_rnn = merge([f_rnn(question_dropout), b_rnn(question_dropout)], mode='concat', concat_axis=-1) + question_dropout = dropout(question_rnn) - # dropout - input_dropout = dropout(input_lstm) - - # cnn - cnns = [Convolution1D(filter_length=filter_length, - nb_filter=self.model_params.get('nb_filters', 1000), - activation=self.model_params.get('conv_activation', 'relu'), - border_mode='same') for filter_length in [2, 3, 5, 7]] - input_cnn = merge([cnn(input_dropout) for cnn in cnns], mode='concat') - - # dropout - input_dropout = dropout(input_cnn) + # could add convolution layer here (as in paper) # maxpooling - maxpool = Lambda(lambda x: K.mean(K.exp(x), axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) - input_pool = maxpool(input_dropout) + maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) + question_pool = maxpool(question_dropout) + + # answer rnn part + f_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, return_sequences=True) + b_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, return_sequences=True, go_backwards=True) + answer_rnn = merge([f_rnn(answer_dropout), b_rnn(answer_dropout)], mode='concat', concat_axis=-1) + answer_dropout = dropout(answer_rnn) + answer_pool = maxpool(answer_dropout) # activation activation = Activation('tanh') - output = activation(input_pool) + question_output = activation(question_pool) + answer_output = activation(answer_pool) - model = Model(input=[input], output=[output]) - return model, model + return question_output, answer_output diff --git a/results.notes b/results.notes index 6fa8d17..cc17138 100644 --- a/results.notes +++ b/results.notes @@ -87,11 +87,11 @@ Model described in paper (Dense + CNN): - Pre-trained the embedding layer via the EmbeddingModel, re-ran - Top 1 precision: - - 0.472 on test 1 + - 0.493 on test 1 - 0.427 on test 2 - 0.470 on dev - MRR: - - 0.595 on test 1 + - 0.620 on test 1 - 0.557 on test 2 - 0.596 on dev diff --git a/word_embeddings.py b/word_embeddings.py deleted file mode 100644 index 9c713b7..0000000 --- a/word_embeddings.py +++ /dev/null @@ -1,28 +0,0 @@ -from gensim.models import Word2Vec -from keras.engine import Layer - -import keras.backend as K - - -class Word2VecEmbedding(Layer): - ''' This layer can be used instead of Keras's Embedding layer, - if word2vec encodings are desired. The performance is generally about - the same, although this isn't as nice of a way to do it. It uses gensim - to train the model, and takes the path to that model in the constructor. - ''' - def __init__(self, model_path, **kwargs): - self.model = Word2Vec.load(model_path) - self.W = K.variable(self.model.syn0) - self.model_dims = self.model.syn0.shape - super(Word2VecEmbedding, self).__init__(**kwargs) - - def build(self, input_shape): - self.trainable_weights = [] - - def get_output_shape_for(self, input_shape): - assert len(input_shape) == 2, 'Must provide a 2D input shape: (n_samples, input_vector)' - return (input_shape[0], input_shape[1], self.model_dims[1]) - - def call(self, x, mask=None): - x = K.maximum(K.minimum(x, self.model_dims[1] - 1), 0) - return K.gather(self.W, x)