diff --git a/attention_lstm.py b/attention_lstm.py index de0a872..e2c5b6c 100644 --- a/attention_lstm.py +++ b/attention_lstm.py @@ -52,6 +52,8 @@ class AttentionLSTM(Recurrent): self.b_regularizer = regularizers.get(b_regularizer) self.dropout_W, self.dropout_U = dropout_W, dropout_U + self.attention_units = list() + if self.dropout_W or self.dropout_U: self.uses_learning_phase = True super(AttentionLSTM, self).__init__(**kwargs) @@ -181,7 +183,6 @@ class AttentionLSTM(Recurrent): return x def step(self, x, states): - h_tm1 = states[0] c_tm1 = states[1] B_U = states[2] diff --git a/insuranceqa.py b/insuranceqa.py index e8bb58e..b940d22 100644 --- a/insuranceqa.py +++ b/insuranceqa.py @@ -67,8 +67,8 @@ def get_eval(f_name): all_answers = [int(i) for i in g.strip().split(' ')] question = convert_from_idxs(q) - q_data.append(pad_sequences([question], maxlen=maxlen, padding='post', truncating='post', value=0)) - a_data.append(pad_sequences([answers[i] for i in all_answers], maxlen=maxlen, padding='post', truncating='post', value=0)) + q_data.append(pad_sequences([question], maxlen=maxlen_question, padding='post', truncating='post', value=0)) + a_data.append(pad_sequences([answers[i] for i in all_answers], maxlen=maxlen_answer, padding='post', truncating='post', value=0)) n_good.append(len(good_answers)) return q_data, a_data, n_good @@ -106,9 +106,9 @@ def get_data(f_name): random.shuffle(combined) q_data[:], ag_data[:], ab_data, targets[:] = zip(*combined) - q_data = pad_sequences(q_data, maxlen=maxlen, padding='post', truncating='post', value=0) - ag_data = pad_sequences(ag_data, maxlen=maxlen, padding='post', truncating='post', value=0) - ab_data = pad_sequences(ab_data, maxlen=maxlen, padding='post', truncating='post', value=0) + q_data = pad_sequences(q_data, maxlen=maxlen_question, padding='post', truncating='post', value=0) + ag_data = pad_sequences(ag_data, maxlen=maxlen_answer, padding='post', truncating='post', value=0) + ab_data = pad_sequences(ab_data, maxlen=maxlen_answer, padding='post', truncating='post', value=0) targets = np.asarray(targets) return q_data, ag_data, ab_data, targets @@ -162,13 +162,14 @@ def get_mrr(model, questions, all_answers, n_good, n_eval=-1): # model parameters n_words = 22354 -maxlen = 40 +maxlen_question = 10 +maxlen_answer = 50 # the model being used print('Generating model') from keras_attention_model import make_model -train_model, test_model = make_model(maxlen, n_words, n_embed_dims=128, n_lstm_dims=256) +train_model, test_model = make_model(maxlen_question, maxlen_answer, n_words, n_embed_dims=128, n_lstm_dims=256) print('Getting data') data_sets = [ @@ -179,16 +180,34 @@ data_sets = [ q_data, ag_data, ab_data, targets = get_data(data_sets[0]) qv_data, avg_data, avb_data, v_targets = get_data(data_sets[1]) +test_model.load_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5')) + # found through experimentation that ~24 epochs generalized the best print('Fitting model') -for i in range(100): - print(i) +for i in range(30): + print('----- %d -----' % i) np.random.shuffle(ab_data) train_model.fit([q_data, ag_data, ab_data], targets, nb_epoch=1, batch_size=128, validation_data=[[qv_data, avg_data, avb_data], v_targets], shuffle=True) + if i % 100 == 0: + train_model.save_weights(os.path.join(models_path, 'iqa_model_for_training_iter_%d.h5' % i), overwrite=True) + test_model.save_weights(os.path.join(models_path, 'iqa_model_for_training_iter_%d.h5' % i), overwrite=True) + train_model.save_weights(os.path.join(models_path, 'iqa_model_for_training.h5'), overwrite=True) test_model.save_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5'), overwrite=True) +test_model.load_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5')) + +import keras.backend as K +get_attention = K.function([test_model.layers[0].input, test_model.layers[1].input], [test_model.layers[3].get_output_at(0)]) +attention = get_attention([q_data[:20], ag_data[:20]])[0] + +for i in range(20): + print('----- %d -----' % i) + print(revert(q_data[i])) + print(revert(ag_data[i])) + print([np.linalg.norm(x) for x in attention[i]]) + # the model actually did really well, predicted correct vs. incorrect answer 85% of the time on the validation set test_model.load_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5')) print('Percent correct: {}'.format(get_accurate_percentage(test_model, q_data, ag_data, ab_data, n_eval='all'))) diff --git a/keras_attention_model.py b/keras_attention_model.py index 6145066..8734899 100644 --- a/keras_attention_model.py +++ b/keras_attention_model.py @@ -15,7 +15,7 @@ from word_embeddings import Word2VecEmbedding models_path = 'models/' -def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128): +def make_model(maxlen_question, maxlen_answer, n_words, n_lstm_dims=141, n_embed_dims=128): from keras.optimizers import RMSprop from attention_lstm import AttentionLSTM @@ -25,39 +25,60 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128): import keras.backend as K # input - question = Input(shape=(maxlen,), dtype='int32') - answer_good = Input(shape=(maxlen,), dtype='int32') - answer_bad = Input(shape=(maxlen,), dtype='int32') + question = Input(shape=(maxlen_question,), dtype='int32', name='question_input') + answer_good = Input(shape=(maxlen_answer,), dtype='int32', name='answer_good_input') + answer_bad = Input(shape=(maxlen_answer,), dtype='int32', name='answer_bad_input') # language model - embedding = Embedding(n_words, n_embed_dims, mask_zero=True) + embedding = Embedding(n_words, n_embed_dims) # forward and backward lstms - f_lstm = LSTM(n_lstm_dims) - b_lstm = LSTM(n_lstm_dims, go_backwards=True) - - # Note: Change concat_axis to 2 if return_sequences=True + f_lstm = LSTM(n_lstm_dims, name='fq_lstm', consume_less='mem', return_sequences=False) + b_lstm = LSTM(n_lstm_dims, name='bq_lstm', go_backwards=True, consume_less='mem', return_sequences=False) # question part q_emb = embedding(question) + q_emb = Convolution1D(nb_filter=64, filter_length=5)(q_emb) + q_emb = MaxPooling1D(pool_length=2)(q_emb) q_fl = f_lstm(q_emb) q_bl = b_lstm(q_emb) - q_out = merge([q_fl, q_bl], mode='concat', concat_axis=1) + q_out = merge([q_fl, q_bl], mode='concat', concat_axis=-1) + q_out = Dropout(0.25)(q_out) + # q_out = Permute((2, 1))(q_out) + # q_out = Convolution1D(nb_filter=64, filter_length=5)(q_out) + # q_out = MaxPooling1D(2)(q_out) + # q_out = Flatten()(q_out) # forward and backward attention lstms (paying attention to q_out) - f_lstm_attention = AttentionLSTM(n_lstm_dims, q_out) - b_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, go_backwards=True) + f_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, consume_less='mem', return_sequences=False) + b_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, go_backwards=True, consume_less='mem', return_sequences=False) + + conv = Convolution1D(nb_filter=64, filter_length=5) # answer part ag_emb = embedding(answer_good) + ag_emb = conv(ag_emb) + ag_emb = MaxPooling1D(pool_length=2)(ag_emb) ag_fl = f_lstm_attention(ag_emb) ag_bl = b_lstm_attention(ag_emb) - ag_out = merge([ag_fl, ag_bl], mode='concat', concat_axis=1) + ag_out = merge([ag_fl, ag_bl], mode='concat', concat_axis=-1) + ag_out = Dropout(0.25)(ag_out) + # ag_out = Permute((2, 1))(ag_out) + # ag_out = conv(ag_out) + # ag_out = MaxPooling1D(2)(ag_out) + # ag_out = Flatten()(ag_out) ab_emb = embedding(answer_bad) + ab_emb = conv(ab_emb) + ab_emb = MaxPooling1D(pool_length=2)(ab_emb) ab_fl = f_lstm_attention(ab_emb) ab_bl = b_lstm_attention(ab_emb) - ab_out = merge([ab_fl, ab_bl], mode='concat', concat_axis=1) + ab_out = merge([ab_fl, ab_bl], mode='concat', concat_axis=-1) + ab_out = Dropout(0.25)(ab_out) + # ab_out = Permute((2, 1))(ab_out) + # ab_out = conv(ab_out) + # ab_out = MaxPooling1D(2)(ab_out) + # ab_out = Flatten()(ab_out) # merge together # note: `cos` refers to "cosine similarity", i.e. similar vectors should go to 1 @@ -65,7 +86,7 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128): good_out = merge([q_out, ag_out], name='good', mode='cos', dot_axes=1) bad_out = merge([q_out, ab_out], name='bad', mode='cos', dot_axes=1) - target = merge([good_out, bad_out], name='target', mode=lambda x: K.maximum(1e-3, 0.2 - x[0] + x[1]), output_shape=lambda x: x[0]) + target = merge([good_out, bad_out], name='target', mode=lambda x: K.maximum(1e-3, 0.3 - x[0] + x[1]), output_shape=lambda x: x[0]) train_model = Model(input=[question, answer_good, answer_bad], output=target) test_model = Model(input=[question, answer_good], output=good_out)