diff --git a/insurance_qa_eval.py b/insurance_qa_eval.py index f848e03..5bcb6a3 100644 --- a/insurance_qa_eval.py +++ b/insurance_qa_eval.py @@ -10,6 +10,8 @@ import pickle from scipy.stats import rankdata +from keras_models import EmbeddingModel + random.seed(42) @@ -23,7 +25,7 @@ class Evaluator: self.path = data_path self.conf = dict() if conf is None else conf self.params = conf.get('training_params', dict()) - self.answers = self.load('answers') + self.answers = self.load('generated') # self.load('answers') self._vocab = None self._reverse_vocab = None self._eval_sets = None @@ -132,6 +134,8 @@ class Evaluator: if save_every is not None and i % save_every == 0: self.save_epoch(model, i) + return val_loss + ##### Evaluation ##### def prog_bar(self, so_far, total, n_bars=20): @@ -226,13 +230,13 @@ if __name__ == '__main__': 'question_len': 20, 'answer_len': 60, 'n_words': 22353, # len(vocabulary) + 1 - 'margin': 0.05, + 'margin': 0.009, 'training_params': { 'save_every': 1, # 'eval_every': 1, 'batch_size': 128, - 'nb_epoch': 1000, + 'nb_epoch': 100, 'validation_split': 0.2, 'optimizer': 'adam', # 'optimizer': Adam(clip_norm=0.1), @@ -245,7 +249,7 @@ if __name__ == '__main__': }, 'model_params': { - 'n_embed_dims': 100, + 'n_embed_dims': 1000, 'n_hidden': 200, # convolution @@ -255,11 +259,11 @@ if __name__ == '__main__': # recurrent 'n_lstm_dims': 141, # * 2 - 'initial_embed_weights': np.load('word2vec_100_dim.embeddings'), + # 'initial_embed_weights': np.load('word2vec_100_dim.embeddings'), }, 'similarity_params': { - 'mode': 'gesd', + 'mode': 'cosine', 'gamma': 1, 'c': 1, 'd': 2, @@ -269,8 +273,7 @@ if __name__ == '__main__': evaluator = Evaluator(conf) ##### Define model ###### - from seq2seq.answer_to_question import EmbeddingRNNModel - model = EmbeddingRNNModel(conf) + model = EmbeddingModel(conf) optimizer = conf.get('training_params', dict()).get('optimizer', 'adam') model.compile(optimizer=optimizer) @@ -284,8 +287,9 @@ if __name__ == '__main__': # train the model # evaluator.load_epoch(model, 54) - evaluator.train(model) + best_loss = evaluator.train(model) # evaluate mrr for a particular epoch - # evaluator.load_epoch(model, 54) - # evaluator.get_mrr(model, evaluate_all=True) + evaluator.load_epoch(model, best_loss['epoch']) + # evaluator.load_epoch(model, 15) + evaluator.get_mrr(model, evaluate_all=True) diff --git a/seq2seq/answer_to_question.py b/seq2seq/answer_to_question.py index a64bf36..da4cd5b 100644 --- a/seq2seq/answer_to_question.py +++ b/seq2seq/answer_to_question.py @@ -11,7 +11,7 @@ import random import numpy as np from keras.engine import Input from keras.layers import RepeatVector, TimeDistributed, Dense, Activation, merge, GRU, Embedding, regularizers, Lambda, \ - Dropout + constraints from keras.models import Model import keras.backend as K @@ -34,6 +34,9 @@ class InsuranceQA: def load(self, name): return pickle.load(open(os.path.join(data_path, name), 'rb')) + def save(self, obj, name): + pickle.dump(obj, open(os.path.join(data_path, name), 'wb')) + class VocabularyTable: def __init__(self, words): self.words = sorted(set(words)) @@ -63,6 +66,8 @@ class InsuranceQA: def get_model(question_maxlen, answer_maxlen, vocab_len, n_hidden, load_save=False): answer = Input(shape=(answer_maxlen,), dtype='int32') embedded = Embedding(input_dim=vocab_len, output_dim=n_hidden, mask_zero=True)(answer) + # answer = Input(shape=(answer_maxlen, vocab_len)) + # embedded = Masking(mask_value=0.)(answer) # encoder rnn encode_rnn = GRU(n_hidden, return_sequences=True, dropout_U=0.2)(embedded) @@ -116,24 +121,22 @@ class EmbeddingRNNModel(LanguageModel): answer_argmax = argmax(answer_inverted) # add embedding layers - # weights = self.model_params.get('initial_embed_weights', None) - # weights = weights if weights is None else [weights] + weights = self.model_params.get('initial_embed_weights', None) + weights = weights if weights is None else [weights] embedding = Embedding(input_dim=self.config['n_words'], output_dim=self.model_params.get('n_embed_dims', 100), - # weights=weights, + # W_regularizer=regularizers.activity_l1(1e-4), + W_constraint=constraints.nonneg(), + dropout=0.5, + weights=weights, mask_zero=True) question_embedding = embedding(question) answer_embedding = embedding(answer_argmax) - # dropout - dropout = Dropout(0.5) - question_dropout = dropout(question_embedding) - answer_dropout = dropout(answer_embedding) - # maxpooling maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) - question_maxpool = maxpool(question_dropout) - answer_maxpool = maxpool(answer_dropout) + question_maxpool = maxpool(question_embedding) + answer_maxpool = maxpool(answer_embedding) # activation activation = Activation('tanh') @@ -190,18 +193,27 @@ if __name__ == '__main__': model = get_model(question_maxlen=question_maxlen, answer_maxlen=answer_maxlen, vocab_len=len(qa.vocab) + 1, n_hidden=256, load_save=True) - print('Training model...') - for iteration in range(1, nb_iteration + 1): - print('\n' + '-' * 50 + '\nIteration %d' % iteration) - model.fit_generator(gen, samples_per_epoch=100 * batch_size, nb_epoch=nb_epoch) - model.save_weights(model_save, overwrite=True) + # print('Training model...') + # for iteration in range(1, nb_iteration + 1): + # print('\n' + '-' * 50 + '\nIteration %d' % iteration) + # model.fit_generator(gen, samples_per_epoch=100 * batch_size, nb_epoch=nb_epoch) + # model.save_weights(model_save, overwrite=True) + # + # # test this iteration on some sample data + # x, y = next(test_gen) + # pred = model.predict(x, verbose=0) + # y = y[0] + # x = x[0] + # for i in range(n_test): + # print('Answer: {}'.format(qa.table.decode(x[i], calc_argmax=False))) + # print(' Expected: {}'.format(qa.table.decode(y[i]))) + # print(' Predicted: {}'.format(qa.table.decode(pred[i]))) - # test this iteration on some sample data - x, y = next(test_gen) - pred = model.predict(x, verbose=0) - y = y[0] - x = x[0] - for i in range(n_test): - print('Answer: {}'.format(qa.table.decode(x[i], calc_argmax=False))) - print(' Expected: {}'.format(qa.table.decode(y[i]))) - print(' Predicted: {}'.format(qa.table.decode(pred[i]))) + print('Saving data points...') + generated = dict() + for key, answer in answers.items(): + print('\r%d / %d' % (key, len(answers)), end = '') + output = model.predict(qa.table.encode([qa.vocab[x] for x in answer], answer_maxlen, one_hot=False).reshape((1, answer_maxlen))) + argmax = np.argmax(output, axis=-1)[0] + generated[key] = answer + qa.save(generated, 'generated')