From 8946dbccf40b7a55fe5b87b00b7bf1bf892ad03c Mon Sep 17 00:00:00 2001 From: codekansas Date: Mon, 2 May 2016 20:30:28 -0400 Subject: [PATCH] ran some more trials --- insurance_qa_eval.py | 71 ++++++++++++++++++++++++++++++++++++-------- keras_models.py | 26 ++++++++++++---- results.notes | 58 ++++++++++++++++-------------------- 3 files changed, 104 insertions(+), 51 deletions(-) diff --git a/insurance_qa_eval.py b/insurance_qa_eval.py index f7ac877..52dbc9b 100644 --- a/insurance_qa_eval.py +++ b/insurance_qa_eval.py @@ -107,7 +107,7 @@ class Evaluator: # random.shuffle(bad_answers) bad_answers = self.pada(random.sample(self.answers.values(), len(good_answers))) - print('Epoch %d :: ' % i, end='') + print('Epoch %d :: ' % (i+1), end='') self.print_time() model.fit([questions, good_answers, bad_answers], nb_epoch=1, batch_size=batch_size, validation_split=split) @@ -119,32 +119,45 @@ class Evaluator: ##### Evaluation ##### + def prog_bar(self, so_far, total, n_bars=20): + n_complete = int(so_far * n_bars / total) + if n_complete >= n_bars - 1: + print('\r[' + '=' * n_bars + ']', end='') + else: + s = '\r[' + '=' * (n_complete - 1) + '>' + '.' * (n_bars - n_complete) + ']' + print(s, end='') + def eval_sets(self): if self._eval_sets is None: self._eval_sets = dict([(s, self.load(s)) for s in ['dev', 'test1', 'test2']]) return self._eval_sets - def get_mrr(self, model): + def get_mrr(self, model, evaluate_all=False): top1s = list() mrrs = list() for name, data in self.eval_sets().items(): - self.print_time() - print('----- %s -----' % name) + if evaluate_all: + self.print_time() + print('----- %s -----' % name) random.shuffle(data) - if 'n_eval' in self.params: + if not evaluate_all and 'n_eval' in self.params: data = data[:self.params['n_eval']] c_1, c_2 = 0, 0 - for d in data: + c = 0 + for i, d in enumerate(data): + if evaluate_all: + self.prog_bar(i, len(data)) + answers = self.pada([self.answers[i] for i in d['good'] + d['bad']]) question = self.padq([d['question']] * len(d['good'] + d['bad'])) n_good = len(d['good']) - sims = model.predict([question, answers], batch_size=300).flatten() + sims = model.predict([question, answers], batch_size=500).flatten() r = rankdata(sims, method='max') max_r = np.argmax(r) @@ -158,12 +171,32 @@ class Evaluator: del data - print('Top-1 Precision: %f' % top1) - print('MRR: %f' % mrr) + if evaluate_all: + print('Top-1 Precision: %f' % top1) + print('MRR: %f' % mrr) top1s.append(top1) mrrs.append(mrr) + # rerun the evaluation if above some threshold + if not evaluate_all: + print('Top-1 Precision: {}'.format(top1s)) + print('MRR: {}'.format(mrrs)) + evaluate_all_threshold = self.params.get('evaluate_all_threshold', dict()) + evaluate_mode = evaluate_all_threshold.get('mode', 'all') + mrr_theshold = evaluate_all_threshold.get('mrr', 1) + top1_threshold = evaluate_all_threshold.get('top1', 1) + + if evaluate_mode == 'any': + evaluate_all = evaluate_all or any([x > top1_threshold for x in top1s]) + evaluate_all = evaluate_all or any([x > mrr_theshold for x in mrrs]) + else: + evaluate_all = evaluate_all or all([x > top1_threshold for x in top1s]) + evaluate_all = evaluate_all or all([x > mrr_theshold for x in mrrs]) + + if evaluate_all: + return self.get_mrr(model, evaluate_all=True) + return top1s, mrrs if __name__ == '__main__': @@ -175,16 +208,21 @@ if __name__ == '__main__': 'training_params': { 'save_every': 1, - # 'eval_every': 20, + 'eval_every': 1, 'batch_size': 128, 'nb_epoch': 1000, 'validation_split': 0.2, 'optimizer': 'adam', - # 'n_eval': 20, + 'n_eval': 20, + + 'evaluate_all_threshold': { + 'mode': 'any', + 'top1': 0.55, + }, }, 'model_params': { - 'n_embed_dims': 1000, + 'n_embed_dims': 100, 'n_hidden': 200, # convolution @@ -210,9 +248,16 @@ if __name__ == '__main__': optimizer = conf.get('training_params', dict()).get('optimizer', 'adam') model.compile(optimizer=optimizer) + # load pre-trained embedding layer + import numpy as np + weights = np.load('models/embedding_100_dim.h5') + language_model = model.prediction_model.layers[2] + language_model.layers[2].set_weights([weights]) + # train the model + evaluator.load_epoch(model, 10) evaluator.train(model) # evaluate mrr for a particular epoch - # evaluator.load_epoch(model, -1) + # evaluator.load_epoch(model, 10) # evaluator.get_mrr(model) diff --git a/keras_models.py b/keras_models.py index a67c962..2c09326 100644 --- a/keras_models.py +++ b/keras_models.py @@ -136,12 +136,12 @@ class LanguageModel: return self.prediction_model.predict(x, **kwargs) def save_weights(self, file_name, **kwargs): - assert self.training_model is not None, 'Must compile the model before saving weights' - self.training_model.save_weights(file_name, **kwargs) + assert self.prediction_model is not None, 'Must compile the model before saving weights' + self.prediction_model.save_weights(file_name, **kwargs) def load_weights(self, file_name, **kwargs): - assert self.training_model is not None, 'Must compile the model loading weights' - self.training_model.load_weights(file_name, **kwargs) + assert self.prediction_model is not None, 'Must compile the model loading weights' + self.prediction_model.load_weights(file_name, **kwargs) class EmbeddingModel(LanguageModel): @@ -181,6 +181,8 @@ class ConvolutionModel(LanguageModel): return merge([cnn(input) for cnn in cnns], mode='concat'), cnns def build(self): + assert self.config['question_len'] == self.config['answer_len'] + question, answer = self._get_inputs() # add embedding layers @@ -188,8 +190,12 @@ class ConvolutionModel(LanguageModel): question_embedding = embedding(question) answer_embedding = embedding(answer) + # turn off layer updating + embedding.params = [] + embedding.updates = [] + # dropout - dropout = Dropout(0.5) + dropout = Dropout(0.25) question_dropout = dropout(question_embedding) answer_dropout = dropout(answer_embedding) @@ -198,6 +204,10 @@ class ConvolutionModel(LanguageModel): question_dense = dense(question_dropout) answer_dense = dense(answer_dropout) + # regularization + question_dense = ActivityRegularization(l2=0.0001)(question_dense) + answer_dense = ActivityRegularization(l2=0.0001)(answer_dense) + # dropout question_dropout = dropout(question_dense) answer_dropout = dropout(answer_dense) @@ -206,10 +216,14 @@ class ConvolutionModel(LanguageModel): cnns = [Convolution1D(filter_length=filter_length, nb_filter=self.model_params.get('nb_filters', 1000), activation=self.model_params.get('conv_activation', 'relu'), - border_mode='same') for filter_length in [2, 3, 4, 5]] + border_mode='same') for filter_length in [2, 3, 5, 7]] question_cnn = merge([cnn(question_dropout) for cnn in cnns], mode='concat') answer_cnn = merge([cnn(answer_dropout) for cnn in cnns], mode='concat') + # regularization + question_cnn = ActivityRegularization(l2=0.0001)(question_cnn) + answer_cnn = ActivityRegularization(l2=0.0001)(answer_cnn) + # dropout question_dropout = dropout(question_cnn) answer_dropout = dropout(answer_cnn) diff --git a/results.notes b/results.notes index 7a61ab0..6fa8d17 100644 --- a/results.notes +++ b/results.notes @@ -4,32 +4,6 @@ Single-layer bi-LSTM with max pooling, 40 words per sentence, loss margin of 0.2 - Using the pure embedding layer worked better than using the Word2Vec model (gave MRR ~0.09) CNN + ALSTM model seems to have good loss - - MRR ~0.33 (I think) - -CNN + ALSTM - - Example error: - - Question: be home insurance negotiable - - Desired answer: I not sure exactly what you mean but if you mean go without then no a mortgage company will not allow - that and if you own outright it will be a foolish choice go without if you mean price negotiation (ranked 177) - - Highest-rank answer: no homeowner be not mandatory in Pennsylvania however if you be apply for any type of homeowner - loan most bank will require you purchase insurance process the loan the bank will also ask be list as the lien holder if - - - Example error: - - Question: what be Suze Orman advice on long term care insurance - - Desired answer: Suze Orman recommend long-term care insurance if it fit and 1 can medically qualify for it she be - currently take care of her 90 year old mother who refuse to letSuze buy long-term care insurance on her when she can - (rank ~460) - - Highest-rank answer: a good age buy long-term care insurance when you be in a good financial position and have some - extra disposable income also if your health be good you will save thousand dollar potentially in premium need specific - number how about - - - Example error: - - Question: can you get Life Insurance on someone else - - Desired answer: you can get life insurance on someone else if you have an insurable interest with them an example of - insurable interest be this : if a nonrelative owe you money you can take out a life insurance policy on them (rank 498) - - Highest-rank answer: I assume when you ask Doe someone have a life insurance policy on me , you be ask about someone - other than your parent as an adult no one can take a policy out on you without you give your - - In general, the model predicts the topics well, but doesn't necessarily match it well with the question. This might be due to the masking issue. - MRR: 0.30957 @@ -42,18 +16,14 @@ Pure CNN Model: - After 120 epochs, InsuranceQA MRR = 0.530 and Top-1 Precision = 0.401 - Decreased slightly with further training due to overfitting - Mixing together different filter lengths improved it to MRR = 0.56 and Top-1 Precision = 0.44 - - - Example error: - - Question: how much do Disability Income insurance cost - - Desired answer: disability income insurance can cost between $50 and $5,000 per month yes that be a ridiculously big range but there be so many variable that determine the rate , I shall cover all base a full-time broker / agent can quickly ask you a few question and determine the good plan for the most affordable price - - Highest-rank answer: along with what Peggy and Steve mention make sure whoever you choose as your agent present you with multiple quote for disability insurance too often consumer think the cheap policy be good but it have likely that the cheap policy offer poor coverage your agent assume he or she do not represent just 1 company shall be able provide you different quote that reflect different type of coverage ( e.g. different benefit period , elimination period , different company , etc. - - Rank of best answer: 347.0 + - Validation loss around 6e-4 (lower than maxpooling) underperformed compared to maxpooling Embedding + MaxPooling: - I can't believe this model performed so well. It blew the other ones out of the water, and trains ridiculously quickly. - Test 1: Top-1 Precision = 0.4922, MRR = 0.6239 - Test 2: Top-1 Precision = 0.4817, MRR = 0.6110 - Dev: Top-1 Precision = 0.4950, MRR = 0.6244 + - With 100 dimensions, Top-1 Precision = 0.281, MRR = 0.417 on test 1 - Adding more embedding dimensions (beyond 1000) didn't lead to an improvement - Converted after about 20 epochs - Validation loss was around 7e-4 (with margin of 0.009) @@ -106,3 +76,27 @@ Model described in paper (Dense + CNN): - 0.343 on dev - Why won't this train better :( + - Best validation loss was about 0.0011 (for margin of 0.009) + - After increasing the number of parameters/embedding size the model improved a bit + - Top 1 precision: + - 0.327 on test 1 + - 0.302 on test 2 + - MRR: + - 0.459 on test 1 + - 0.433 on test 2 + + - Pre-trained the embedding layer via the EmbeddingModel, re-ran + - Top 1 precision: + - 0.472 on test 1 + - 0.427 on test 2 + - 0.470 on dev + - MRR: + - 0.595 on test 1 + - 0.557 on test 2 + - 0.596 on dev + +1000 embed dims + 2000 CNN filters (total over 4 lengths): + - Loss ~6e-4 for margin of 0.009 + - Test 1: 0.409 Top-1 Precision, 0.543 MRR + - Test 2: 0.376 Top-1 Precision, 0.507 MRR +