From a78094112e55e1021fe94140adebece545a1bea5 Mon Sep 17 00:00:00 2001 From: codekansas Date: Wed, 20 Apr 2016 10:58:21 -0400 Subject: [PATCH] merged --- keras_attention_model.py | 54 ++++++++-------------------------------- 1 file changed, 10 insertions(+), 44 deletions(-) diff --git a/keras_attention_model.py b/keras_attention_model.py index e73aa53..6145066 100644 --- a/keras_attention_model.py +++ b/keras_attention_model.py @@ -30,15 +30,11 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128): answer_bad = Input(shape=(maxlen,), dtype='int32') # language model - embedding = Embedding(n_words, n_embed_dims) - # embedding = Word2VecEmbedding(os.path.join(models_path, 'word2vec.model')) + embedding = Embedding(n_words, n_embed_dims, mask_zero=True) # forward and backward lstms - f_lstm = LSTM(n_lstm_dims, return_sequences=True) - b_lstm = LSTM(n_lstm_dims, go_backwards=True, return_sequences=True) - - f_lstm_2 = LSTM(n_lstm_dims, return_sequences=True) - b_lstm_2 = LSTM(n_lstm_dims, go_backwards=True, return_sequences=True) + f_lstm = LSTM(n_lstm_dims) + b_lstm = LSTM(n_lstm_dims, go_backwards=True) # Note: Change concat_axis to 2 if return_sequences=True @@ -46,51 +42,22 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128): q_emb = embedding(question) q_fl = f_lstm(q_emb) q_bl = b_lstm(q_emb) - q_out = merge([q_fl, q_bl], mode='concat', concat_axis=2) - - q_out_fl = f_lstm_2(q_out) - q_out_bl = b_lstm_2(q_out) - q_out = merge([q_out_fl, q_out_bl], mode='concat', concat_axis=2) - - q_out = Permute((2, 1))(q_out) - q_out = MaxPooling1D(2 * n_lstm_dims)(q_out) - q_out = Flatten()(q_out) + q_out = merge([q_fl, q_bl], mode='concat', concat_axis=1) # forward and backward attention lstms (paying attention to q_out) - f_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, return_sequences=True) - b_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, go_backwards=True, return_sequences=True) - - f_lstm_3 = LSTM(n_lstm_dims, return_sequences=True) - b_lstm_3 = LSTM(n_lstm_dims, go_backwards=True, return_sequences=True) - - conv = Convolution1D(64, 5) + f_lstm_attention = AttentionLSTM(n_lstm_dims, q_out) + b_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, go_backwards=True) # answer part ag_emb = embedding(answer_good) ag_fl = f_lstm_attention(ag_emb) ag_bl = b_lstm_attention(ag_emb) - ag_out = merge([ag_fl, ag_bl], mode='concat', concat_axis=2) - - ag_out_fl = f_lstm_3(ag_out) - ag_out_bl = b_lstm_3(ag_out) - ag_out = merge([ag_out_fl, ag_out_bl], mode='concat', concat_axis=2) - - ag_out = Permute((2, 1))(ag_out) - ag_out = MaxPooling1D(2 * n_lstm_dims)(ag_out) - ag_out = Flatten()(ag_out) + ag_out = merge([ag_fl, ag_bl], mode='concat', concat_axis=1) ab_emb = embedding(answer_bad) ab_fl = f_lstm_attention(ab_emb) ab_bl = b_lstm_attention(ab_emb) - ab_out = merge([ab_fl, ab_bl], mode='concat', concat_axis=2) - - ab_out_fl = f_lstm_3(ab_out) - ab_out_bl = b_lstm_3(ab_out) - ab_out = merge([ab_out_fl, ab_out_bl], mode='concat', concat_axis=2) - - ab_out = Permute((2, 1))(ab_out) - ab_out = MaxPooling1D(2 * n_lstm_dims)(ab_out) - ab_out = Flatten()(ab_out) + ab_out = merge([ab_fl, ab_bl], mode='concat', concat_axis=1) # merge together # note: `cos` refers to "cosine similarity", i.e. similar vectors should go to 1 @@ -117,7 +84,7 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128): def loss(y_true, y_pred): return y_pred - # unfortunately, the hinge loss approach means the "accura cy" metric isn't very valuable + # unfortunately, the hinge loss approach means the "accuracy" metric isn't very valuable metrics = [] train_model.compile(optimizer=optimizer, loss=loss, metrics=metrics) @@ -127,14 +94,13 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128): if __name__ == '__main__': # get the data set - maxlen = 200 # words + maxlen = 40 # words from utils.get_data import get_data_set, create_dictionary_from_qas dic = create_dictionary_from_qas() targets, questions, good_answers, bad_answers, n_dims = get_data_set(maxlen) - ### THIS MODEL PERFORMS WELL ON THE TEST SET train_model, test_model = make_model(maxlen, n_dims) print('Fitting model')