From a853407a3a9796b3f4045f2ee8d92b5e7c2188c9 Mon Sep 17 00:00:00 2001 From: codekansas Date: Fri, 6 May 2016 03:17:07 -0400 Subject: [PATCH] added answer_to_question generative model --- answer_to_question.py | 130 +++++++++++++++++++++++++++++++++++++ attention_lstm.py | 36 ++++++---- insurance_qa_embeddings.py | 1 - insurance_qa_eval.py | 40 +++++++----- keras_models.py | 33 ++++------ 5 files changed, 190 insertions(+), 50 deletions(-) create mode 100644 answer_to_question.py diff --git a/answer_to_question.py b/answer_to_question.py new file mode 100644 index 0000000..aa9f7be --- /dev/null +++ b/answer_to_question.py @@ -0,0 +1,130 @@ +from __future__ import print_function +import numpy as np + +import os +from keras.engine import Input +from keras.layers import LSTM, RepeatVector, TimeDistributed, Dense, Activation +from keras.models import Model + +# can remove this depending on ide... +os.environ['INSURANCE_QA'] = '/media/moloch/HHD/MachineLearning/data/insuranceQA/pyenc' + +import sys + +try: + import cPickle as pickle +except: + import pickle + + +class InsuranceQA: + def __init__(self): + try: + data_path = os.environ['INSURANCE_QA'] + except KeyError: + print("INSURANCE_QA is not set. Set it to your clone of https://github.com/codekansas/insurance_qa_python") + sys.exit(1) + self.path = data_path + self.vocab = self.load('vocabulary') + self.table = InsuranceQA.VocabularyTable(self.vocab.values()) + + def load(self, name): + return pickle.load(open(os.path.join(self.path, name), 'rb')) + + class VocabularyTable: + ''' Identical to CharacterTable from Keras example ''' + def __init__(self, words): + self.words = sorted(set(words)) + self.words_indices = dict((c, i) for i, c in enumerate(self.words)) + self.indices_words = dict((i, c) for i, c in enumerate(self.words)) + + def encode(self, sentence, maxlen): + indices = np.zeros((maxlen, len(self.words))) + for i, w in enumerate(sentence): + if i == maxlen: break + indices[i, self.words_indices[w]] = 1 + return indices + + def decode(self, indices, calc_argmax=True): + if calc_argmax: + indices = indices.argmax(axis=-1) + return ' '.join(self.indices_words[x] for x in indices) + +def get_model(question_maxlen, answer_maxlen, vocab_len, n_hidden): + answer = Input(shape=(answer_maxlen, vocab_len)) + # answer = Masking(mask_value=0.)(answer) + + # encoder rnn + encode_rnn = LSTM(n_hidden, return_sequences=False)(answer) + + # can add more layers + for i in range(2): + encode_rnn = LSTM(n_hidden, return_sequences=True)(encode_rnn) + + # repeat it maxlen times + repeat_encoding = RepeatVector(question_maxlen)(encode_rnn) + + # decoder rnn + decode_rnn = LSTM(n_hidden, return_sequences=True)(repeat_encoding) + + # can add more layers + for i in range(2): + decode_rnn = LSTM(n_hidden, return_sequences=True)(decode_rnn) + + # output + dense = TimeDistributed(Dense(vocab_len))(decode_rnn) + softmax = Activation('softmax')(dense) + + # compile the model + model = Model([answer], [softmax]) + model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) + + return model + +if __name__ == '__main__': + question_maxlen, answer_maxlen = 10, 40 + + qa = InsuranceQA() + batch_size = 50 + n_test = 10 + + print('Generating data...') + answers = qa.load('answers') + questions = qa.load('train') + + def gen_questions(batch_size): + while True: + i = 0 + question_idx = np.zeros(shape=(batch_size, question_maxlen, len(qa.vocab))) + answer_idx = np.zeros(shape=(batch_size, answer_maxlen, len(qa.vocab))) + for s in questions: + a = s['answers'][0] + answer = qa.table.encode([qa.vocab[x] for x in answers[a]], answer_maxlen) + question = qa.table.encode([qa.vocab[x] for x in s['question']], question_maxlen) + answer_idx[i] = answer + question_idx[i] = question + i += 1 + if i == batch_size: + yield ([answer_idx], [question_idx]) + i = 0 + + gen = gen_questions(batch_size) + test_gen = gen_questions(n_test) + + print('Generating model...') + model = get_model(question_maxlen=question_maxlen, answer_maxlen=answer_maxlen, + vocab_len=len(qa.vocab), n_hidden=128) + + print('Training model...') + for iteration in range(1, 200): + print() + print('-' * 50) + print('Iteration', iteration) + model.fit_generator(gen, samples_per_epoch=100*batch_size, nb_epoch=10) + + x, y = next(test_gen) + y = y[0] + pred = model.predict(x, verbose=0) + for i in range(n_test): + print('Expected: {}'.format(qa.table.decode(y[i]))) + print('Predicted: {}'.format(qa.table.decode(pred[i]))) diff --git a/attention_lstm.py b/attention_lstm.py index a37ad16..53a1dca 100644 --- a/attention_lstm.py +++ b/attention_lstm.py @@ -1,12 +1,18 @@ from __future__ import absolute_import from keras import backend as K -from keras.layers import LSTM +from keras.layers import LSTM, activations class AttentionLSTM(LSTM): - def __init__(self, output_dim, attention_vec, **kwargs): + def __init__(self, output_dim, attention_vec, attn_activation='tanh', + attn_inner_activation='tanh', single_attn=False, + n_attention_dim=None, **kwargs): self.attention_vec = attention_vec + self.attn_activation = activations.get(attn_activation) + self.attn_inner_activation = activations.get(attn_inner_activation) + self.single_attention_param = single_attn + self.n_attention_dim = output_dim if n_attention_dim is None else n_attention_dim super(AttentionLSTM, self).__init__(output_dim, **kwargs) @@ -26,12 +32,14 @@ class AttentionLSTM(LSTM): name='{}_U_m'.format(self.name)) self.b_m = K.zeros((self.output_dim,), name='{}_b_m'.format(self.name)) - # self.U_s = self.inner_init((self.output_dim, self.output_dim), - # name='{}_U_s'.format(self.name)) - # self.b_s = K.zeros((self.output_dim,), name='{}_b_s'.format(self.name)) - self.U_s = self.inner_init((self.output_dim, 1), - name='{}_U_s'.format(self.name)) - self.b_s = K.zeros((1,), name='{}_b_s'.format(self.name)) + if self.single_attention_param: + self.U_s = self.inner_init((self.output_dim, 1), + name='{}_U_s'.format(self.name)) + self.b_s = K.zeros((1,), name='{}_b_s'.format(self.name)) + else: + self.U_s = self.inner_init((self.output_dim, self.output_dim), + name='{}_U_s'.format(self.name)) + self.b_s = K.zeros((self.output_dim,), name='{}_b_s'.format(self.name)) self.trainable_weights += [self.U_a, self.U_m, self.U_s, self.b_a, self.b_m, self.b_s] @@ -43,13 +51,15 @@ class AttentionLSTM(LSTM): h, [h, c] = super(AttentionLSTM, self).step(x, states) attention = states[4] - m = K.tanh(K.dot(h, self.U_a) * attention + self.b_a) + m = self.attn_inner_activation(K.dot(h, self.U_a) * attention + self.b_a) # Intuitively it makes more sense to use a sigmoid (was getting some NaN problems # which I think might have been caused by the exponential function -> gradients blow up) - # s = K.exp(K.dot(m, self.U_s) + self.b_s) - s = K.tanh(K.dot(m, self.U_s) + self.b_s) - h = h * K.repeat_elements(s, self.output_dim, axis=1) - # h = h * s + s = self.attn_activation(K.dot(m, self.U_s) + self.b_s) + + if self.single_attention_param: + h = h * K.repeat_elements(s, self.output_dim, axis=1) + else: + h = h * s return h, [h, c] diff --git a/insurance_qa_embeddings.py b/insurance_qa_embeddings.py index d7fb2d3..5928632 100644 --- a/insurance_qa_embeddings.py +++ b/insurance_qa_embeddings.py @@ -3,7 +3,6 @@ from __future__ import print_function import os import sys import random -from time import strftime, gmtime import pickle diff --git a/insurance_qa_eval.py b/insurance_qa_eval.py index c63834a..cae8905 100644 --- a/insurance_qa_eval.py +++ b/insurance_qa_eval.py @@ -1,23 +1,30 @@ from __future__ import print_function import os +# can remove this depending on ide... +os.environ['INSURANCE_QA'] = '/media/moloch/HHD/MachineLearning/data/insuranceQA/pyenc' + import sys import random from time import strftime, gmtime import pickle -from keras.optimizers import Adam, RMSprop +from keras.optimizers import RMSprop from scipy.stats import rankdata from keras_models import * random.seed(42) - class Evaluator: - def __init__(self, path, conf=None): - self.path = path + def __init__(self, conf=None): + try: + data_path = os.environ['INSURANCE_QA'] + except KeyError: + print("INSURANCE_QA is not set. Set it to your clone of https://github.com/codekansas/insurance_qa_python") + sys.exit(1) + self.path = data_path self.conf = dict() if conf is None else conf self.params = conf.get('training_params', dict()) self.answers = self.load('answers') @@ -107,6 +114,11 @@ class Evaluator: # random.shuffle(bad_answers) bad_answers = self.pada(random.sample(self.answers.values(), len(good_answers))) + # shuffle questions + zipped = zip(questions, good_answers) + random.shuffle(zipped) + questions[:], good_answers[:] = zip(*zipped) + print('Epoch %d :: ' % (i+1), end='') self.print_time() model.fit([questions, good_answers, bad_answers], nb_epoch=1, batch_size=batch_size, validation_split=split) @@ -200,30 +212,24 @@ class Evaluator: return top1s, mrrs if __name__ == '__main__': - try: - data_path = os.environ['INSURANCE_QA'] - except KeyError: - print("INSURANCE_QA is not set. Set it to your clone of https://github.com/codekansas/insurance_qa_python") - sys.exit(1) - conf = { - 'question_len': 20, - 'answer_len': 100, + 'question_len': 30, + 'answer_len': 150, 'n_words': 22353, # len(vocabulary) + 1 - 'margin': 0.02, + 'margin': 0.2, 'training_params': { 'save_every': 1, 'eval_every': 1, 'batch_size': 128, 'nb_epoch': 1000, - 'validation_split': 0.2, + 'validation_split': 0.1, 'optimizer': RMSprop(clip_norm=0.1), # Adam(clip_norm=0.1), 'n_eval': 20, 'evaluate_all_threshold': { 'mode': 'all', - 'top1': 0.5, + 'top1': 0.4, }, }, @@ -247,7 +253,7 @@ if __name__ == '__main__': } } - evaluator = Evaluator(data_path, conf) + evaluator = Evaluator(conf) ##### Define model ###### model = AttentionModel(conf) @@ -269,7 +275,7 @@ if __name__ == '__main__': language_model.layers[2].set_weights([weights]) # train the model - # evaluator.load_epoch(model, 25) + # evaluator.load_epoch(model, 225) evaluator.train(model) # evaluate mrr for a particular epoch diff --git a/keras_models.py b/keras_models.py index 78d97f7..97df8d0 100644 --- a/keras_models.py +++ b/keras_models.py @@ -4,7 +4,7 @@ from abc import abstractmethod from keras.engine import Input from keras.layers import merge, Embedding, Dropout, Convolution1D, Lambda, Activation, LSTM, Dense, TimeDistributed, \ - ActivityRegularization, Flatten + ActivityRegularization from keras import backend as K from keras.models import Model @@ -240,7 +240,7 @@ class AttentionModel(LanguageModel): answer = self.get_answer() # add embedding layers - embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 100)) + embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 100), mask_zero=False) question_embedding = embedding(question) answer_embedding = embedding(answer) @@ -256,28 +256,23 @@ class AttentionModel(LanguageModel): # question rnn part f_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True) b_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True, go_backwards=True) - question_rnn = merge([f_rnn(question_dropout), b_rnn(question_dropout)], mode='concat', concat_axis=-1) - question_dropout = dropout(question_rnn) - - # regularize - regularize = ActivityRegularization(l2=0.0001) - question_dropout = regularize(question_dropout) - - # could add convolution layer here (as in paper) + question_f_rnn = f_rnn(question_dropout) + question_b_rnn = b_rnn(question_dropout) + question_f_dropout = dropout(question_f_rnn) + question_b_dropout = dropout(question_b_rnn) # maxpooling maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) - question_pool = maxpool(question_dropout) + question_pool = merge([maxpool(question_f_dropout), maxpool(question_b_dropout)], mode='concat', concat_axis=-1) # answer rnn part - f_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, return_sequences=True) - b_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, return_sequences=True, go_backwards=True) - # f_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True) - # b_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True, go_backwards=True) - answer_rnn = merge([f_rnn(answer_dropout), b_rnn(answer_dropout)], mode='concat', concat_axis=-1) - answer_dropout = dropout(answer_rnn) - answer_dropout = regularize(answer_dropout) - answer_pool = maxpool(answer_dropout) + f_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, single_attn=True, return_sequences=True) + b_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, single_attn=True, return_sequences=True, go_backwards=True) + answer_f_rnn = f_rnn(answer_dropout) + answer_b_rnn = b_rnn(answer_dropout) + answer_f_dropout = dropout(answer_f_rnn) + answer_b_dropout = dropout(answer_b_rnn) + answer_pool = merge([maxpool(answer_f_dropout), maxpool(answer_b_dropout)], mode='concat', concat_axis=-1) # activation activation = Activation('tanh')