From 51223f2ec2262a0e4f614df24125c31e4aaf6e29 Mon Sep 17 00:00:00 2001 From: codekansas Date: Sat, 30 Apr 2016 02:34:59 -0400 Subject: [PATCH] made everything make sense --- insurance_qa_eval.py | 7 ++- keras_attention_model.py | 109 ------------------------------------- keras_attention_one_way.py | 104 ----------------------------------- keras_cnn_model.py | 95 -------------------------------- keras_models.py | 10 ++-- results.notes | 5 ++ 6 files changed, 15 insertions(+), 315 deletions(-) delete mode 100644 keras_attention_model.py delete mode 100644 keras_attention_one_way.py delete mode 100644 keras_cnn_model.py diff --git a/insurance_qa_eval.py b/insurance_qa_eval.py index 0e22b92..e117e7b 100644 --- a/insurance_qa_eval.py +++ b/insurance_qa_eval.py @@ -155,8 +155,8 @@ class Evaluator: if __name__ == '__main__': conf = { - 'question_len': 100, - 'answer_len': 100, + 'question_len': 10, + 'answer_len': 40, 'n_words': 22353, # len(vocabulary) + 1 'margin': 0.009, @@ -184,13 +184,14 @@ if __name__ == '__main__': 'mode': 'cosine', 'gamma': 1, 'c': 1, + 'd': 2, } } evaluator = Evaluator(data_path, conf) ##### Define model ###### - model = EmbeddingModel(conf) + model = RecurrentModel(conf) model.compile(optimizer='adam') evaluator.train(model) diff --git a/keras_attention_model.py b/keras_attention_model.py deleted file mode 100644 index f7c8142..0000000 --- a/keras_attention_model.py +++ /dev/null @@ -1,109 +0,0 @@ -from __future__ import print_function - -############## -# Make model # -############## -import os - -from keras.layers import Dropout, Embedding, Lambda, Convolution1D -from keras.optimizers import RMSprop - -from attention_lstm import AttentionLSTM - -from keras.layers import Input, LSTM, merge -from keras.models import Model -import keras.backend as K - -models_path = 'models/' - - -def make_model(maxlen_question, maxlen_answer, n_words, n_embed_dims=128): - n_lstm_dims = 141 - - # input - question = Input(shape=(maxlen_question,), dtype='int32', name='question_input') - answer_good = Input(shape=(maxlen_answer,), dtype='int32', name='answer_good_input') - answer_bad = Input(shape=(maxlen_answer,), dtype='int32', name='answer_bad_input') - - # language model - embedding = Embedding(n_words, n_embed_dims) - maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) - - # forward and backward lstms - f_lstm = LSTM(n_lstm_dims, name='fq_lstm', consume_less='mem', return_sequences=True) - b_lstm = LSTM(n_lstm_dims, name='bq_lstm', go_backwards=True, consume_less='mem', return_sequences=True) - - # question part - q_emb = embedding(question) - q_emb = Dropout(0.25)(q_emb) - q_fl = f_lstm(q_emb) - q_bl = b_lstm(q_emb) - q_out = merge([q_fl, q_bl], mode='concat', concat_axis=-1) - q_out = maxpool(q_out) - - # forward and backward attention lstms (paying attention to q_out) - f_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, consume_less='mem', return_sequences=True) - b_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, go_backwards=True, consume_less='mem', return_sequences=True) - - # answer part - ag_emb = embedding(answer_good) - ag_emb = Dropout(0.25)(ag_emb) - ag_fl = f_lstm_attention(ag_emb) - ag_bl = b_lstm_attention(ag_emb) - ag_out = merge([ag_fl, ag_bl], mode='concat', concat_axis=-1) - ag_out = maxpool(ag_out) - - ab_emb = embedding(answer_bad) - ab_emb = Dropout(0.25)(ab_emb) - ab_fl = f_lstm_attention(ab_emb) - ab_bl = b_lstm_attention(ab_emb) - ab_out = merge([ab_fl, ab_bl], mode='concat', concat_axis=-1) - ab_out = maxpool(ab_out) - - # merge together - # note: `cos` refers to "cosine similarity", i.e. similar vectors should go to 1 - # for training's sake, "abs" limits range to between 0 and 1 (binary classification) - good_out = merge([q_out, ag_out], name='good', mode='cos', dot_axes=1) - bad_out = merge([q_out, ab_out], name='bad', mode='cos', dot_axes=1) - - target = merge([good_out, bad_out], name='target', mode=lambda x: K.maximum(1e-6, 0.009 - x[0] + x[1]), output_shape=lambda x: x[0]) - - train_model = Model(input=[question, answer_good, answer_bad], output=target) - test_model = Model(input=[question, answer_good], output=good_out) - - print('Compiling model...') - - optimizer = RMSprop(lr=0.0001, clipnorm=0.05) - # optimizer = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True, clipgrad=0.1) - - # this is more true to the paper: L = max{0, M - cosine(q, a+) + cosine(q, a-)} - # below, "a" is a list of zeros and "b" is `target` above, i.e. 1 - cosine(q, a+) + cosine(q, a-) - # loss = 'binary_crossentropy' - # loss = 'mse' - # loss = 'hinge' - - def loss(y_true, y_pred): - return y_pred - - # unfortunately, the hinge loss approach means the "accuracy" metric isn't very valuable - metrics = [] - - train_model.compile(optimizer=optimizer, loss=loss, metrics=metrics) - test_model.compile(optimizer=optimizer, loss=loss, metrics=metrics) - - return train_model, test_model - -if __name__ == '__main__': - # get the data set - maxlen = 40 # words - - from utils.get_data import get_data_set, create_dictionary_from_qas - - dic = create_dictionary_from_qas() - targets, questions, good_answers, bad_answers, n_dims = get_data_set(maxlen) - - train_model, test_model = make_model(maxlen, n_dims) - - print('Fitting model') - train_model.fit([questions, good_answers, bad_answers], targets, nb_epoch=5, batch_size=128) - train_model.save_weights(os.path.join(models_path, 'attention_lm_weights.h5'), overwrite=True) diff --git a/keras_attention_one_way.py b/keras_attention_one_way.py deleted file mode 100644 index 9136003..0000000 --- a/keras_attention_one_way.py +++ /dev/null @@ -1,104 +0,0 @@ -from __future__ import print_function - -############## -# Make model # -############## -import os - -from keras.layers import Dropout, Embedding, Lambda, Convolution1D, MaxPooling1D -from keras.optimizers import RMSprop - -from attention_lstm import AttentionLSTM - -from keras.layers import Input, LSTM, merge -from keras.models import Model -import keras.backend as K - -models_path = 'models/' - - -def make_model(maxlen_question, maxlen_answer, n_words, n_embed_dims=128): - n_lstm_dims = 400 - - # input - question = Input(shape=(maxlen_question,), dtype='int32', name='question_input') - answer_good = Input(shape=(maxlen_answer,), dtype='int32', name='answer_good_input') - answer_bad = Input(shape=(maxlen_answer,), dtype='int32', name='answer_bad_input') - - # language model - embedding = Embedding(n_words, n_embed_dims, mask_zero=True) - - # forward and backward lstms - f_lstm = LSTM(n_lstm_dims, name='fq_lstm', consume_less='mem', return_sequences=True) - - def maxpool_func(x, mask=None): - return K.max(x, axis=1, keepdims=False) - maxpool = Lambda(maxpool_func, output_shape=lambda x: (x[0], x[2])) - - # question part - q_emb = embedding(question) - q_emb = Dropout(0.25)(q_emb) - q_out = f_lstm(q_emb) - q_out = maxpool(q_out) - - # forward and backward attention lstms (paying attention to q_out) - f_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, consume_less='mem', return_sequences=True) - - # answer part - ag_emb = embedding(answer_good) - ag_emb = Dropout(0.25)(ag_emb) - ag_out = f_lstm_attention(ag_emb) - ag_out = maxpool(ag_out) - - ab_emb = embedding(answer_bad) - ab_emb = Dropout(0.25)(ab_emb) - ab_out = f_lstm_attention(ab_emb) - ab_out = maxpool(ab_out) - - # merge together - # note: `cos` refers to "cosine similarity", i.e. similar vectors should go to 1 - # for training's sake, "abs" limits range to between 0 and 1 (binary classification) - good_out = merge([q_out, ag_out], name='good', mode='cos', dot_axes=1) - bad_out = merge([q_out, ab_out], name='bad', mode='cos', dot_axes=1) - - target = merge([good_out, bad_out], name='target', mode=lambda x: K.maximum(1e-6, 0.1 - x[0] + x[1]), output_shape=lambda x: x[0]) - - train_model = Model(input=[question, answer_good, answer_bad], output=target) - test_model = Model(input=[question, answer_good], output=good_out) - - print('Compiling model...') - - optimizer = RMSprop(lr=0.0001, clipnorm=0.05) - # optimizer = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True, clipgrad=0.1) - - # this is more true to the paper: L = max{0, M - cosine(q, a+) + cosine(q, a-)} - # below, "a" is a list of zeros and "b" is `target` above, i.e. 1 - cosine(q, a+) + cosine(q, a-) - # loss = 'binary_crossentropy' - # loss = 'mse' - # loss = 'hinge' - - def loss(y_true, y_pred): - return y_pred - - # unfortunately, the hinge loss approach means the "accuracy" metric isn't very valuable - metrics = [] - - train_model.compile(optimizer=optimizer, loss=loss, metrics=metrics) - test_model.compile(optimizer=optimizer, loss=loss, metrics=metrics) - - return train_model, test_model - -if __name__ == '__main__': - # get the data set - maxlen = 40 # words - - from utils.get_data import get_data_set, create_dictionary_from_qas - - dic = create_dictionary_from_qas() - targets, questions, good_answers, bad_answers, n_dims = get_data_set(maxlen) - - train_model, test_model = make_model(maxlen, n_dims) - - print('Fitting model') - train_model.fit([questions, good_answers, bad_answers], targets, nb_epoch=5, batch_size=128) - train_model.save_weights(os.path.join(models_path, 'attention_lm_weights.h5'), overwrite=True) diff --git a/keras_cnn_model.py b/keras_cnn_model.py deleted file mode 100644 index 63cb631..0000000 --- a/keras_cnn_model.py +++ /dev/null @@ -1,95 +0,0 @@ -from __future__ import print_function - -############## -# Make model # -############## -import os - -from keras.engine import Merge -from keras.layers import MaxPooling1D, Dropout, Embedding, Convolution1D, Dense, Lambda, TimeDistributed, \ - ActivityRegularization - -models_path = 'models/' - - -def make_model(maxlen_question, maxlen_answer, n_words, n_embed_dims=128): - from keras.layers import Input, merge - from keras.models import Model - import keras.backend as K - - # input - question = Input(shape=(maxlen_question,), dtype='int32', name='question_input') - answer_good = Input(shape=(maxlen_answer,), dtype='int32', name='answer_good_input') - answer_bad = Input(shape=(maxlen_answer,), dtype='int32', name='answer_bad_input') - - # language model - embedding = Embedding(n_words, n_embed_dims) - - # embedding - q_emb = embedding(question) - ag_emb = embedding(answer_good) - ab_emb = embedding(answer_bad) - - # dense - dense = TimeDistributed(Dense(200, activation='tanh')) - q_dense = dense(q_emb) - ag_dense = dense(ag_emb) - ab_dense = dense(ab_emb) - - # regularlize - q_dense = ActivityRegularization(l2=0.0001)(q_dense) - ag_dense = ActivityRegularization(l2=0.0001)(ag_dense) - ab_dense = ActivityRegularization(l2=0.0001)(ab_dense) - - # dropout - q_dense = Dropout(0.25)(q_dense) - ag_dense = Dropout(0.25)(ag_dense) - ab_dense = Dropout(0.25)(ab_dense) - - # cnn - cnns = [Convolution1D(filter_length=filt, nb_filter=1000, activation='relu', border_mode='same') for filt in [2, 3, 5, 7]] - q_cnn = merge([cnn(q_dense) for cnn in cnns], mode='concat') - ag_cnn = merge([cnn(ag_dense) for cnn in cnns], mode='concat') - ab_cnn = merge([cnn(ab_dense) for cnn in cnns], mode='concat') - - # dropout - q_cnn = Dropout(0.25)(q_cnn) - ag_cnn = Dropout(0.25)(ag_cnn) - ab_cnn = Dropout(0.25)(ab_cnn) - - # maxpooling - # maxpool = MaxPooling1D(pool_length=2) - maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) - q_pool = maxpool(q_cnn) - ag_pool = maxpool(ag_cnn) - ab_pool = maxpool(ab_cnn) - - # tanh - tanh = Lambda(lambda x: K.tanh(x)) - q_out = tanh(q_pool) - ag_out = tanh(ag_pool) - ab_out = tanh(ab_pool) - - # merge together - good_out = merge([q_out, ag_out], mode='cos', dot_axes=1) - bad_out = merge([q_out, ab_out], mode='cos', dot_axes=1) - target = merge([good_out, bad_out], name='target', mode=lambda x: K.maximum(1e-6, 0.009 - x[0] + x[1]), output_shape=lambda x: x[0]) - - train_model = Model(input=[question, answer_good, answer_bad], output=target) - test_model = Model(input=[question, answer_good], output=good_out) - - print('Compiling model...') - - # optimizer = RMSprop(lr=0.01, clipnorm=0.05) - optimizer = 'adam' - - def loss(y_true, y_pred): - return y_pred - - # unfortunately, the hinge loss approach means the "accuracy" metric isn't very valuable - metrics = [] - - train_model.compile(optimizer=optimizer, loss=loss, metrics=metrics) - test_model.compile(optimizer=optimizer, loss=loss, metrics=metrics) - - return train_model, test_model diff --git a/keras_models.py b/keras_models.py index e68304e..5b71bbe 100644 --- a/keras_models.py +++ b/keras_models.py @@ -150,6 +150,8 @@ class LanguageModel: class EmbeddingModel(LanguageModel): + ''' This model actually performs stupidly well ''' + def build(self): input, _ = self._get_inputs() @@ -232,12 +234,12 @@ class RecurrentModel(LanguageModel): input_dropout = dropout(input_embedding) # rnn - # forward_lstm = LSTM(self.config.get('n_lstm_dims', 141), consume_less='mem', return_sequences=False) - # backward_lstm = LSTM(self.config.get('n_lstm_dims', 141), consume_less='mem', return_sequences=False) - # input_lstm = merge([forward_lstm(input_dropout), backward_lstm(input_dropout)], mode='concat', concat_axis=-1) + forward_lstm = LSTM(self.config.get('n_lstm_dims', 141), consume_less='mem', return_sequences=True) + backward_lstm = LSTM(self.config.get('n_lstm_dims', 141), consume_less='mem', return_sequences=True) + input_lstm = merge([forward_lstm(input_dropout), backward_lstm(input_dropout)], mode='concat', concat_axis=-1) # dropout - # input_dropout = dropout(input_lstm) + input_dropout = dropout(input_lstm) # maxpooling maxpool = Lambda(lambda x: K.mean(K.exp(x), axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) diff --git a/results.notes b/results.notes index 4d3f836..5c4ea84 100644 --- a/results.notes +++ b/results.notes @@ -49,4 +49,9 @@ Pure CNN Model: - Highest-rank answer: along with what Peggy and Steve mention make sure whoever you choose as your agent present you with multiple quote for disability insurance too often consumer think the cheap policy be good but it have likely that the cheap policy offer poor coverage your agent assume he or she do not represent just 1 company shall be able provide you different quote that reflect different type of coverage ( e.g. different benefit period , elimination period , different company , etc. - Rank of best answer: 347.0 +Embedding + MaxPooling: + - I can't believe this model performed so well. It blew the other ones out of the water, and trains ridiculously quickly. + - Test 1: Top-1 Precision = 0.4933, MRR = 0.6189 + - Test 2: Top-1 Precision = 0.4606, MRR = 0.5968 + - Dev: Top-1 Precision = 0.4700, MRR = 0.6088