diff --git a/insurance_qa_eval.py b/insurance_qa_eval.py index 4630fd0..f7ac877 100644 --- a/insurance_qa_eval.py +++ b/insurance_qa_eval.py @@ -6,6 +6,7 @@ from time import strftime, gmtime import pickle +from keras.optimizers import Adam from scipy.stats import rankdata from keras_models import * @@ -41,6 +42,17 @@ class Evaluator: self._reverse_vocab = dict((v.lower(), k) for k, v in vocab.items()) return self._reverse_vocab + ##### Loading / saving ##### + + def save_epoch(self, model, epoch): + if not os.path.exists('models/'): + os.makedirs('models/') + model.save_weights('models/weights_epoch_%d.h5' % epoch, overwrite=True) + + def load_epoch(self, model, epoch): + assert os.path.exists('models/weights_epoch_%d.h5' % epoch), 'Weights at epoch %d not found' % epoch + model.load_weights('models/weights_epoch_%d.h5' % epoch) + ##### Converting / reverting ##### def convert(self, words): @@ -75,6 +87,7 @@ class Evaluator: save_every = self.params.get('save_every', None) batch_size = self.params.get('batch_size', 128) nb_epoch = self.params.get('nb_epoch', 10) + split = self.params.get('validation_split', 0) training_set = self.load('train') @@ -96,13 +109,13 @@ class Evaluator: print('Epoch %d :: ' % i, end='') self.print_time() - model.fit([questions, good_answers, bad_answers], nb_epoch=1, batch_size=batch_size) + model.fit([questions, good_answers, bad_answers], nb_epoch=1, batch_size=batch_size, validation_split=split) if eval_every is not None and (i+1) % eval_every == 0: self.get_mrr(model) if save_every is not None and (i+1) % save_every == 0: - model.save_weights('weights.h5') + self.save_epoch(model, (i+1)) ##### Evaluation ##### @@ -131,7 +144,7 @@ class Evaluator: question = self.padq([d['question']] * len(d['good'] + d['bad'])) n_good = len(d['good']) - sims = model.predict([question, answers], verbose=1, batch_size=128).flatten() + sims = model.predict([question, answers], batch_size=300).flatten() r = rankdata(sims, method='max') max_r = np.argmax(r) @@ -155,21 +168,23 @@ class Evaluator: if __name__ == '__main__': conf = { - 'question_len': 20, + 'question_len': 100, 'answer_len': 100, 'n_words': 22353, # len(vocabulary) + 1 - 'margin': 0.1, + 'margin': 0.009, 'training_params': { - 'eval_every': 25, - 'save_every': None, + 'save_every': 1, + # 'eval_every': 20, 'batch_size': 128, - 'nb_epoch': 100, + 'nb_epoch': 1000, + 'validation_split': 0.2, + 'optimizer': 'adam', # 'n_eval': 20, }, 'model_params': { - 'n_embed_dims': 300, + 'n_embed_dims': 1000, 'n_hidden': 200, # convolution @@ -192,6 +207,12 @@ if __name__ == '__main__': ##### Define model ###### model = ConvolutionModel(conf) - model.compile(optimizer='adam') + optimizer = conf.get('training_params', dict()).get('optimizer', 'adam') + model.compile(optimizer=optimizer) + # train the model evaluator.train(model) + + # evaluate mrr for a particular epoch + # evaluator.load_epoch(model, -1) + # evaluator.get_mrr(model) diff --git a/keras_models.py b/keras_models.py index a61f14a..a67c962 100644 --- a/keras_models.py +++ b/keras_models.py @@ -15,23 +15,28 @@ from attention_lstm import AttentionLSTM class LanguageModel: def __init__(self, config): - self.question = Input(shape=(config['question_len'],), dtype='int32') - self.answer_good = Input(shape=(config['answer_len'],), dtype='int32') - self.answer_bad = Input(shape=(config['answer_len'],), dtype='int32') + self.question = Input(shape=(config['question_len'],), dtype='int32', name='question') + self.answer_good = Input(shape=(config['answer_len'],), dtype='int32', name='answer_good') + self.answer_bad = Input(shape=(config['answer_len'],), dtype='int32', name='answer_bad') self.config = config self.model_params = config.get('model_params', dict()) self.similarity_params = config.get('similarity_params', dict()) + # initialize a bunch of variables that will be set later self._models = None self._similarities = None + self._inputs = None + self._qa_model = None self.training_model = None self.prediction_model = None def _get_inputs(self): - return [Input(shape=(self.config['question_len'],), dtype='int32', name='question'), - Input(shape=(self.config['answer_len'],), dtype='int32', name='answer')] + if self._inputs is None: + self._inputs = [Input(shape=(self.config['question_len'],), dtype='int32', name='question'), + Input(shape=(self.config['answer_len'],), dtype='int32', name='answer')] + return self._inputs @abstractmethod def build(self): @@ -92,45 +97,35 @@ class LanguageModel: else: raise Exception('Invalid similarity: {}'.format(similarity)) - def get_similarities(self): + def get_qa_model(self): if self._models is None: self._models = self.build() - assert len(self._models) == 2, 'build() should make question and answer language models' - if self._similarities is None: - question_model, answer_model = self._models - - answers_use_question = len(answer_model.internal_input_shapes) == 2 - - question = question_model(self.question, self.answer_good) - - if answers_use_question: - good = answer_model([self.question, self.answer_good]) - bad = answer_model([self.question, self.answer_bad]) - else: - good = answer_model([self.answer_good]) - bad = answer_model([self.answer_bad]) + if self._qa_model is None: + question_output, answer_output = self._models similarity = self.get_similarity() - good_sim = merge([question, good], mode=similarity, output_shape=lambda x: x[:-1]) - bad_sim = merge([question, bad], mode=similarity, output_shape=lambda x: x[:-1]) + qa_model = merge([question_output, answer_output], mode=similarity, output_shape=lambda x: x[:-1]) - self._similarities = [good_sim, bad_sim] + self._qa_model = Model(input=self._get_inputs(), output=[qa_model]) - return self._similarities + return self._qa_model def compile(self, optimizer, **kwargs): - similarities = self.get_similarities() + qa_model = self.get_qa_model() - loss = merge(similarities, + good_output = qa_model([self.question, self.answer_good]) + bad_output = qa_model([self.question, self.answer_bad]) + + loss = merge([good_output, bad_output], mode=lambda x: K.maximum(1e-6, self.config['margin'] - x[0] + x[1]), output_shape=lambda x: x[0]) self.training_model = Model(input=[self.question, self.answer_good, self.answer_bad], output=loss) self.training_model.compile(loss=lambda y_true, y_pred: y_pred, optimizer=optimizer, **kwargs) - self.prediction_model = Model(input=[self.question, self.answer_good], output=similarities[0]) - self.prediction_model.compile(loss=lambda y_true, y_pred: y_pred, optimizer=optimizer, **kwargs) + self.prediction_model = Model(input=[self.question, self.answer_good], output=good_output) + self.prediction_model.compile(loss='binary_crossentropy', optimizer=optimizer, **kwargs) def fit(self, x, **kwargs): assert self.training_model is not None, 'Must compile the model before fitting data' @@ -150,30 +145,30 @@ class LanguageModel: class EmbeddingModel(LanguageModel): - ''' This model actually performs stupidly well ''' - def build(self): - input, _ = self._get_inputs() + question, answer = self._get_inputs() # add embedding layers embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 141)) - input_embedding = embedding(input) + question_embedding = embedding(question) + answer_embedding = embedding(answer) # dropout dropout = Dropout(0.5) - input_dropout = dropout(input_embedding) + question_dropout = dropout(question_embedding) + answer_dropout = dropout(answer_embedding) # maxpooling maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) - input_pool = maxpool(input_dropout) + question_maxpool = maxpool(question_dropout) + answer_maxpool = maxpool(answer_dropout) # activation activation = Activation('tanh') - output = activation(input_pool) + question_output = activation(question_maxpool) + answer_output = activation(answer_maxpool) - model = Model(input=[input], output=[output]) - - return model, model + return question_output, answer_output class ConvolutionModel(LanguageModel): @@ -189,37 +184,31 @@ class ConvolutionModel(LanguageModel): question, answer = self._get_inputs() # add embedding layers - embedding_1 = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 100)) - embedding_2 = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 100)) - question_embedding = embedding_1(question) - answer_embedding = embedding_2(answer) - - # use the same word embeddings for both the question and answer models - # embedding_1.set_weights(embedding_2.get_weights()) + embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 100)) + question_embedding = embedding(question) + answer_embedding = embedding(answer) # dropout - dropout = Dropout(0.25) + dropout = Dropout(0.5) question_dropout = dropout(question_embedding) answer_dropout = dropout(answer_embedding) - # # dense - # dense_1 = TimeDistributed(Dense(self.model_params.get('n_hidden', 200), activation='tanh')) - # dense_2 = TimeDistributed(Dense(self.model_params.get('n_hidden', 200), activation='tanh')) - # question_dense = dense_1(question_dropout) - # answer_dense = dense_2(answer_dropout) - # - # # use the same weights for both layers - # dense_1.set_weights(dense_2.get_weights()) - # - # question_dropout = dropout(question_dense) - # answer_dropout = dropout(answer_dense) + # dense + dense = TimeDistributed(Dense(self.model_params.get('n_hidden', 200), activation='tanh')) + question_dense = dense(question_dropout) + answer_dense = dense(answer_dropout) + + # dropout + question_dropout = dropout(question_dense) + answer_dropout = dropout(answer_dense) # cnn - question_cnn, cnns_1 = self.mixed_filter_lengths(question_dropout, [2, 3, 5, 7]) - answer_cnn, cnns_2 = self.mixed_filter_lengths(answer_dropout, [2, 3, 5, 7]) - - for a, b in zip(cnns_1, cnns_2): - b.set_weights(a.get_weights()) + cnns = [Convolution1D(filter_length=filter_length, + nb_filter=self.model_params.get('nb_filters', 1000), + activation=self.model_params.get('conv_activation', 'relu'), + border_mode='same') for filter_length in [2, 3, 4, 5]] + question_cnn = merge([cnn(question_dropout) for cnn in cnns], mode='concat') + answer_cnn = merge([cnn(answer_dropout) for cnn in cnns], mode='concat') # dropout question_dropout = dropout(question_cnn) @@ -235,9 +224,7 @@ class ConvolutionModel(LanguageModel): question_output = activation(question_pool) answer_output = activation(answer_pool) - question_model = Model(input=[question], output=[question_output]) - answer_model = Model(input=[answer], output=[answer_output]) - return question_model, answer_model + return question_output, answer_output class RecurrentModel(LanguageModel): @@ -280,60 +267,3 @@ class RecurrentModel(LanguageModel): model = Model(input=[input], output=[output]) return model, model - - -class AttentionModel(LanguageModel): - def build(self): - question, answer = self._get_inputs() - - # add embedding layers - embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 141)) - question_embedding = embedding(question) - - a_embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 141)) - answer_embedding = embedding(answer) - - a_embedding.set_weights(embedding.get_weights()) - - # dropout - dropout = Dropout(0.5) - question_dropout = dropout(question_embedding) - answer_dropout = dropout(answer_embedding) - - # rnn - forward_lstm = LSTM(self.config.get('n_lstm_dims', 141), consume_less='mem', return_sequences=True) - backward_lstm = LSTM(self.config.get('n_lstm_dims', 141), consume_less='mem', return_sequences=True) - question_lstm = merge([forward_lstm(question_dropout), backward_lstm(question_dropout)], mode='concat', concat_axis=-1) - - # dropout - question_dropout = dropout(question_lstm) - - # maxpooling - maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) - question_pool = maxpool(question_dropout) - - # activation - activation = Activation('tanh') - question_output = activation(question_pool) - - question_model = Model(input=[question], output=[question_output]) - - # attentional rnn - forward_lstm = AttentionLSTM(self.config.get('n_lstm_dims', 141), question_output, consume_less='mem', return_sequences=True) - backward_lstm = AttentionLSTM(self.config.get('n_lstm_dims', 141), question_output, consume_less='mem', return_sequences=True) - answer_lstm = merge([forward_lstm(answer_dropout), backward_lstm(answer_dropout)], mode='concat', concat_axis=-1) - - # dropout - answer_dropout = dropout(answer_lstm) - - # maxpooling - maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) - answer_pool = maxpool(answer_dropout) - - # activation - activation = Activation('tanh') - answer_output = activation(answer_pool) - - answer_model = Model(input=[question, answer], output=[answer_output]) - - return question_model, answer_model diff --git a/models b/models new file mode 120000 index 0000000..cda6728 --- /dev/null +++ b/models @@ -0,0 +1 @@ +/media/moloch/HHD/MachineLearning/models/insuranceqa \ No newline at end of file diff --git a/results.notes b/results.notes index 31f6515..7a61ab0 100644 --- a/results.notes +++ b/results.notes @@ -51,9 +51,49 @@ Pure CNN Model: Embedding + MaxPooling: - I can't believe this model performed so well. It blew the other ones out of the water, and trains ridiculously quickly. - - Test 1: Top-1 Precision = 0.4933, MRR = 0.6189 - - Test 2: Top-1 Precision = 0.4606, MRR = 0.5968 - - Dev: Top-1 Precision = 0.4700, MRR = 0.6088 + - Test 1: Top-1 Precision = 0.4922, MRR = 0.6239 + - Test 2: Top-1 Precision = 0.4817, MRR = 0.6110 + - Dev: Top-1 Precision = 0.4950, MRR = 0.6244 + - Adding more embedding dimensions (beyond 1000) didn't lead to an improvement + - Converted after about 20 epochs + - Validation loss was around 7e-4 (with margin of 0.009) + - Configuration: + conf = { + 'question_len': 100, + 'answer_len': 100, + 'n_words': 22353, # len(vocabulary) + 1 + 'margin': 0.009, + + 'training_params': { + 'save_every': 1, + 'eval_every': 20, + 'batch_size': 128, + 'nb_epoch': 1000, + 'validation_split': 0.2, + 'optimizer': 'adam', + # 'n_eval': 20, + }, + + 'model_params': { + 'n_embed_dims': 1000, + 'n_hidden': 200, + + # convolution + 'nb_filters': 1000, + 'conv_activation': 'relu', + + # recurrent + 'n_lstm_dims': 300, + }, + + 'similarity_params': { + 'mode': 'cosine', + 'gamma': 1, + 'c': 1, + 'd': 2, + } + } + Model described in paper (Dense + CNN): - Top 1 precision: @@ -64,10 +104,5 @@ Model described in paper (Dense + CNN): - 0.328 on test 1 - 0.319 on test 2 - 0.343 on dev - -Plain Embedding + CNN: - - Top 1 precision: - - 0.364 on test 1 - - MRR: - - 0.517 on test 1 + - Why won't this train better :(