diff --git a/insurance_qa_eval.py b/insurance_qa_eval.py index cae8905..618a363 100644 --- a/insurance_qa_eval.py +++ b/insurance_qa_eval.py @@ -10,7 +10,7 @@ from time import strftime, gmtime import pickle -from keras.optimizers import RMSprop +from keras.optimizers import RMSprop, Adam from scipy.stats import rankdata from keras_models import * @@ -108,26 +108,32 @@ class Evaluator: good_answers = self.pada(good_answers) # bad_answers = self.pada(random.sample(self.answers.values(), len(good_answers))) - for i in range(nb_epoch): + val_loss = {'loss': 1., 'epoch': 0} + + for i in range(1, nb_epoch): # bad_answers = np.roll(good_answers, random.randint(10, len(questions) - 10)) # bad_answers = good_answers.copy() # random.shuffle(bad_answers) bad_answers = self.pada(random.sample(self.answers.values(), len(good_answers))) # shuffle questions - zipped = zip(questions, good_answers) - random.shuffle(zipped) - questions[:], good_answers[:] = zip(*zipped) + # zipped = zip(questions, good_answers) + # random.shuffle(zipped) + # questions[:], good_answers[:] = zip(*zipped) - print('Epoch %d :: ' % (i+1), end='') + print('Epoch %d :: ' % i, end='') self.print_time() - model.fit([questions, good_answers, bad_answers], nb_epoch=1, batch_size=batch_size, validation_split=split) + hist = model.fit([questions, good_answers, bad_answers], nb_epoch=1, batch_size=batch_size, validation_split=split) - if eval_every is not None and (i+1) % eval_every == 0: + if hist.history['val_loss'][0] < val_loss['loss']: + val_loss = {'loss': hist.history['val_loss'][0], 'epoch': i} + print('Best: Loss = {}, Epoch = {}'.format(val_loss['loss'], val_loss['epoch'])) + + if eval_every is not None and i % eval_every == 0: self.get_mrr(model) - if save_every is not None and (i+1) % save_every == 0: - self.save_epoch(model, (i+1)) + if save_every is not None and i % save_every == 0: + self.save_epoch(model, i) ##### Evaluation ##### @@ -160,13 +166,13 @@ class Evaluator: c_1, c_2 = 0, 0 - c = 0 for i, d in enumerate(data): if evaluate_all: self.prog_bar(i, len(data)) - answers = self.pada([self.answers[i] for i in d['good'] + d['bad']]) - question = self.padq([d['question']] * len(d['good'] + d['bad'])) + indices = d['good'] + d['bad'] + answers = self.pada([self.answers[i] for i in indices]) + question = self.padq([d['question']] * len(indices)) n_good = len(d['good']) sims = model.predict([question, answers], batch_size=500).flatten() @@ -175,6 +181,10 @@ class Evaluator: max_r = np.argmax(r) max_n = np.argmax(r[:n_good]) + # print(' '.join(self.revert(d['question']))) + # print(' '.join(self.revert(self.answers[indices[max_r]]))) + # print(' '.join(self.revert(self.answers[indices[max_n]]))) + c_1 += 1 if max_r == max_n else 0 c_2 += 1 / float(r[max_r] - r[max_n] + 1) @@ -204,7 +214,7 @@ class Evaluator: evaluate_all = evaluate_all or any([x >= mrr_theshold for x in mrrs]) else: evaluate_all = evaluate_all or all([x >= top1_threshold for x in top1s]) - evaluate_all = evaluate_all and all([x >= mrr_theshold for x in mrrs]) + evaluate_all = evaluate_all or all([x >= mrr_theshold for x in mrrs]) if evaluate_all: return self.get_mrr(model, evaluate_all=True) @@ -213,19 +223,20 @@ class Evaluator: if __name__ == '__main__': conf = { - 'question_len': 30, - 'answer_len': 150, + 'question_len': 100, + 'answer_len': 100, 'n_words': 22353, # len(vocabulary) + 1 - 'margin': 0.2, + 'margin': 0.009, 'training_params': { 'save_every': 1, - 'eval_every': 1, + # 'eval_every': 1, 'batch_size': 128, 'nb_epoch': 1000, - 'validation_split': 0.1, - 'optimizer': RMSprop(clip_norm=0.1), # Adam(clip_norm=0.1), - 'n_eval': 20, + 'validation_split': 0.2, + 'optimizer': 'adam', + # 'optimizer': Adam(clip_norm=0.1), + # 'n_eval': 100, 'evaluate_all_threshold': { 'mode': 'all', @@ -238,15 +249,17 @@ if __name__ == '__main__': 'n_hidden': 200, # convolution - 'nb_filters': 1000, + 'nb_filters': 1000, # * 4 'conv_activation': 'relu', # recurrent 'n_lstm_dims': 141, + + 'initial_embed_weights': np.load('word2vec_100_dim.embeddings'), }, 'similarity_params': { - 'mode': 'gesd', + 'mode': 'cosine', 'gamma': 1, 'c': 1, 'd': 2, @@ -256,28 +269,24 @@ if __name__ == '__main__': evaluator = Evaluator(conf) ##### Define model ###### - model = AttentionModel(conf) + model = ConvolutionModel(conf) optimizer = conf.get('training_params', dict()).get('optimizer', 'adam') model.compile(optimizer=optimizer) import numpy as np # save embedding layer + # evaluator.load_epoch(model, 33) # embedding_layer = model.prediction_model.layers[2].layers[2] # evaluator.load_epoch(model, 100) # evaluator.train(model) # weights = embedding_layer.get_weights()[0] - # np.save(open('models/embedding_200_dim.h5', 'wb'), weights) - - # load pre-trained embedding layer - weights = np.load('word2vec_100_dim.embeddings') - language_model = model.prediction_model.layers[2] - language_model.layers[2].set_weights([weights]) + # np.save(open('models/embedding_1000_dim.h5', 'wb'), weights) # train the model - # evaluator.load_epoch(model, 225) + # evaluator.load_epoch(model, 6) evaluator.train(model) # evaluate mrr for a particular epoch - # evaluator.load_epoch(model, 53) + # evaluator.load_epoch(model, 22) # evaluator.get_mrr(model, evaluate_all=True) diff --git a/keras_models.py b/keras_models.py index 97df8d0..79c803f 100644 --- a/keras_models.py +++ b/keras_models.py @@ -129,7 +129,7 @@ class LanguageModel: def fit(self, x, **kwargs): assert self.training_model is not None, 'Must compile the model before fitting data' y = np.zeros(shape=x[0].shape[:1]) - self.training_model.fit(x, y, **kwargs) + return self.training_model.fit(x, y, **kwargs) def predict(self, x, **kwargs): return self.prediction_model.predict(x, **kwargs) @@ -149,7 +149,12 @@ class EmbeddingModel(LanguageModel): answer = self.get_answer() # add embedding layers - embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 141)) + weights = self.model_params.get('initial_embed_weights', None) + weights = weights if weights is None else [weights] + embedding = Embedding(input_dim=self.config['n_words'], + output_dim=self.model_params.get('n_embed_dims', 100), + weights=weights, + mask_zero=True) question_embedding = embedding(question) answer_embedding = embedding(answer) @@ -172,6 +177,8 @@ class EmbeddingModel(LanguageModel): class ConvolutionModel(LanguageModel): + ### Validation loss at Epoch 65: 2.4e-6 + def build(self): assert self.config['question_len'] == self.config['answer_len'] @@ -179,13 +186,17 @@ class ConvolutionModel(LanguageModel): answer = self.get_answer() # add embedding layers - embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 100)) + weights = self.model_params.get('initial_embed_weights', None) + weights = weights if weights is None else [weights] + embedding = Embedding(input_dim=self.config['n_words'], + output_dim=self.model_params.get('n_embed_dims', 100), + weights=weights) question_embedding = embedding(question) answer_embedding = embedding(answer) # turn off layer updating - embedding.params = [] - embedding.updates = [] + # embedding.params = [] + # embedding.updates = [] # dropout dropout = Dropout(0.25) @@ -213,10 +224,6 @@ class ConvolutionModel(LanguageModel): question_cnn = merge([cnn(question_dropout) for cnn in cnns], mode='concat') answer_cnn = merge([cnn(answer_dropout) for cnn in cnns], mode='concat') - # regularization - question_cnn = ActivityRegularization(l2=0.0001)(question_cnn) - answer_cnn = ActivityRegularization(l2=0.0001)(answer_cnn) - # dropout question_dropout = dropout(question_cnn) answer_dropout = dropout(answer_cnn) @@ -240,7 +247,12 @@ class AttentionModel(LanguageModel): answer = self.get_answer() # add embedding layers - embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 100), mask_zero=False) + weights = self.model_params.get('initial_embed_weights', None) + weights = weights if weights is None else [weights] + embedding = Embedding(input_dim=self.config['n_words'], + output_dim=self.model_params.get('n_embed_dims', 100), + weights=weights, + mask_zero=True) question_embedding = embedding(question) answer_embedding = embedding(answer) @@ -254,8 +266,9 @@ class AttentionModel(LanguageModel): answer_dropout = dropout(answer_embedding) # question rnn part - f_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True) - b_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True, go_backwards=True) + f_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True, consume_less='mem') + b_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True, consume_less='mem', + go_backwards=True) question_f_rnn = f_rnn(question_dropout) question_b_rnn = b_rnn(question_dropout) question_f_dropout = dropout(question_f_rnn) @@ -266,8 +279,10 @@ class AttentionModel(LanguageModel): question_pool = merge([maxpool(question_f_dropout), maxpool(question_b_dropout)], mode='concat', concat_axis=-1) # answer rnn part - f_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, single_attn=True, return_sequences=True) - b_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, single_attn=True, return_sequences=True, go_backwards=True) + f_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, single_attn=True, + return_sequences=True, consume_less='mem') + b_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, single_attn=True, + return_sequences=True, consume_less='mem', go_backwards=True) answer_f_rnn = f_rnn(answer_dropout) answer_b_rnn = b_rnn(answer_dropout) answer_f_dropout = dropout(answer_f_rnn) diff --git a/seq2seq/__init__.py b/seq2seq/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/answer_to_question.py b/seq2seq/answer_to_question.py similarity index 58% rename from answer_to_question.py rename to seq2seq/answer_to_question.py index 07e1216..bbe4954 100644 --- a/answer_to_question.py +++ b/seq2seq/answer_to_question.py @@ -1,10 +1,18 @@ +''' +The training model learns to generate a "question" which contains all the same +words as the original question. So it isn't really learning a sequence, but the +result is interesting. +''' + from __future__ import print_function import numpy as np import os from keras.engine import Input -from keras.layers import LSTM, RepeatVector, TimeDistributed, Dense, Activation, Masking, merge +from keras.layers import LSTM, RepeatVector, TimeDistributed, Dense, Activation, Masking, merge, activations, Lambda, \ + ActivityRegularization from keras.models import Model +import keras.backend as K # can remove this depending on ide... os.environ['INSURANCE_QA'] = '/media/moloch/HHD/MachineLearning/data/insuranceQA/pyenc' @@ -45,47 +53,54 @@ class InsuranceQA: indices[i, self.words_indices[w]] = 1 return indices - def decode(self, indices, calc_argmax=True, noise=0): + def decode(self, indices, calc_argmax=True, noise=0.2): if calc_argmax: - indices = indices + np.random.rand(*indices.shape) * noise - indices = indices.argmax(axis=-1) + indices = [self.sample(i, noise=noise) for i in indices] return ' '.join(self.indices_words[x] for x in indices) + def sample(self, index, noise=0.2): + index = np.log(index) / noise + index = np.exp(index) / np.sum(np.exp(index)) + index = np.argmax(np.random.multinomial(1, index, 1)) + return index + def get_model(question_maxlen, answer_maxlen, vocab_len, n_hidden): answer = Input(shape=(answer_maxlen, vocab_len)) masked = Masking(mask_value=0.)(answer) # encoder rnn - encode_rnn = LSTM(n_hidden, return_sequences=True)(masked) - encode_rnn = LSTM(n_hidden, return_sequences=False)(encode_rnn) + encode_rnn = LSTM(n_hidden, return_sequences=True, dropout_U=0.2)(masked) + encode_rnn = LSTM(n_hidden, return_sequences=False, dropout_U=0.2)(encode_rnn) - encode_brnn = LSTM(n_hidden, return_sequences=True, go_backwards=True)(masked) - encode_brnn = LSTM(n_hidden, return_sequences=False, go_backwards=True)(encode_brnn) + encode_brnn = LSTM(n_hidden, return_sequences=True, go_backwards=True, dropout_U=0.2)(masked) + encode_brnn = LSTM(n_hidden, return_sequences=False, go_backwards=True, dropout_U=0.2)(encode_brnn) # repeat it maxlen times - repeat_encoding = RepeatVector(question_maxlen)(encode_rnn) + repeat_encoding_rnn = RepeatVector(question_maxlen)(encode_rnn) + repeat_encoding_brnn = RepeatVector(question_maxlen)(encode_brnn) # decoder rnn - decode_rnn = LSTM(n_hidden, return_sequences=True)(repeat_encoding) - decode_rnn = LSTM(n_hidden, return_sequences=True)(decode_rnn) + decode_rnn = LSTM(n_hidden, return_sequences=True, dropout_U=0.2, dropout_W=0.5)(repeat_encoding_rnn) + decode_rnn = LSTM(n_hidden, return_sequences=True, dropout_U=0.2)(decode_rnn) - decode_brnn = LSTM(n_hidden, return_sequences=True, go_backwards=True)(repeat_encoding) - decode_brnn = LSTM(n_hidden, return_sequences=True, go_backwards=True)(decode_brnn) + decode_brnn = LSTM(n_hidden, return_sequences=True, go_backwards=True, dropout_U=0.2, dropout_W=0.5)(repeat_encoding_brnn) + decode_brnn = LSTM(n_hidden, return_sequences=True, go_backwards=True, dropout_U=0.2)(decode_brnn) merged_output = merge([decode_rnn, decode_brnn], mode='concat', concat_axis=-1) # output dense = TimeDistributed(Dense(vocab_len))(merged_output) - softmax = Activation('softmax')(dense) + regularized = ActivityRegularization(l2=1)(dense) + softmax = Activation('softmax')(regularized) - # compile the model + # compile the prediction model model = Model([answer], [softmax]) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) return model if __name__ == '__main__': - question_maxlen, answer_maxlen = 10, 40 + question_maxlen, answer_maxlen = 20, 60 qa = InsuranceQA() batch_size = 50 @@ -93,30 +108,37 @@ if __name__ == '__main__': print('Generating data...') answers = qa.load('answers') - questions = qa.load('train') - def gen_questions(batch_size): + def gen_questions(batch_size, test=False): + if test: + questions = qa.load('test1') + else: + questions = qa.load('train') while True: i = 0 question_idx = np.zeros(shape=(batch_size, question_maxlen, len(qa.vocab))) answer_idx = np.zeros(shape=(batch_size, answer_maxlen, len(qa.vocab))) for s in questions: - a = s['answers'][0] - answer = qa.table.encode([qa.vocab[x] for x in answers[a]], answer_maxlen) - question = qa.table.encode([qa.vocab[x] for x in s['question']], question_maxlen) - answer_idx[i] = answer - question_idx[i] = question - i += 1 - if i == batch_size: - yield ([answer_idx], [question_idx]) - i = 0 + if test: + ans = s['good'] + else: + ans = s['answers'] + for a in ans: + answer = qa.table.encode([qa.vocab[x] for x in answers[a]], answer_maxlen) + question = qa.table.encode([qa.vocab[x] for x in s['question']], question_maxlen) + question = np.amax(question, axis=0, keepdims=False) + answer_idx[i] = answer + question_idx[i] = question + i += 1 + if i == batch_size: + yield ([answer_idx], [question_idx]) + i = 0 gen = gen_questions(batch_size) - test_gen = gen_questions(n_test) + test_gen = gen_questions(n_test, test=True) print('Generating model...') - model = get_model(question_maxlen=question_maxlen, answer_maxlen=answer_maxlen, - vocab_len=len(qa.vocab), n_hidden=128) + model = get_model(question_maxlen=question_maxlen, answer_maxlen=answer_maxlen, vocab_len=len(qa.vocab), n_hidden=128) print('Training model...') for iteration in range(1, 200): @@ -128,7 +150,7 @@ if __name__ == '__main__': x, y = next(test_gen) y = y[0] pred = model.predict(x, verbose=0) - for noise in [0, 0.1, 0.2]: # not sure what noise values would be good + for noise in [0.2, 0.5, 1.0, 1.2]: # not sure what noise values would be good print(' Noise: {}'.format(noise)) for i in range(n_test): print(' Expected: {}'.format(qa.table.decode(y[i]))) diff --git a/seq2seq/seq2seq.py b/seq2seq/seq2seq.py new file mode 100644 index 0000000..658d914 --- /dev/null +++ b/seq2seq/seq2seq.py @@ -0,0 +1,30 @@ +import keras.backend as K +from keras.engine import Input +from keras.layers import LSTM, TimeDistributed, Dense + + +class Seq2Seq: + def __init__(self, encode_seq_length, decode_seq_length, n_symbols, **params): + self.input = Input(shape=(encode_seq_length, n_symbols,)) + + self.encode_seq_length = encode_seq_length + self.decode_seq_length = decode_seq_length + self.n_symbols = n_symbols + self.params = params + + encoder = self.build_encoder(self.input) + decoder = self.build_decoder(encoder) + + def get_param(self, param, default): + if param in self.params: + return self.params[param] + print('Could not find param "{}" in params: Using default value {}'.format(param, default)) + return default + + def build_encoder(self, input): + lstm = LSTM(self.get_param('n_lstm_dims', 100), return_sequences=False)(input) + dense = Dense(self.n_symbols)(lstm) + return dense + + def build_decoder(self, input): + pass \ No newline at end of file