merged

keras_attention_model.py

@@ -30,15 +30,11 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128):
     answer_bad = Input(shape=(maxlen,), dtype='int32')
 
     # language model
-    embedding = Embedding(n_words, n_embed_dims)
-    # embedding = Word2VecEmbedding(os.path.join(models_path, 'word2vec.model'))
+    embedding = Embedding(n_words, n_embed_dims, mask_zero=True)
 
     # forward and backward lstms
-    f_lstm = LSTM(n_lstm_dims, return_sequences=True)
-    b_lstm = LSTM(n_lstm_dims, go_backwards=True, return_sequences=True)
-
-    f_lstm_2 = LSTM(n_lstm_dims, return_sequences=True)
-    b_lstm_2 = LSTM(n_lstm_dims, go_backwards=True, return_sequences=True)
+    f_lstm = LSTM(n_lstm_dims)
+    b_lstm = LSTM(n_lstm_dims, go_backwards=True)
 
     # Note: Change concat_axis to 2 if return_sequences=True
 
@@ -46,51 +42,22 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128):
     q_emb = embedding(question)
     q_fl = f_lstm(q_emb)
     q_bl = b_lstm(q_emb)
-    q_out = merge([q_fl, q_bl], mode='concat', concat_axis=2)
-
-    q_out_fl = f_lstm_2(q_out)
-    q_out_bl = b_lstm_2(q_out)
-    q_out = merge([q_out_fl, q_out_bl], mode='concat', concat_axis=2)
-
-    q_out = Permute((2, 1))(q_out)
-    q_out = MaxPooling1D(2 * n_lstm_dims)(q_out)
-    q_out = Flatten()(q_out)
+    q_out = merge([q_fl, q_bl], mode='concat', concat_axis=1)
 
     # forward and backward attention lstms (paying attention to q_out)
-    f_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, return_sequences=True)
-    b_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, go_backwards=True, return_sequences=True)
-
-    f_lstm_3 = LSTM(n_lstm_dims, return_sequences=True)
-    b_lstm_3 = LSTM(n_lstm_dims, go_backwards=True, return_sequences=True)
-
-    conv = Convolution1D(64, 5)
+    f_lstm_attention = AttentionLSTM(n_lstm_dims, q_out)
+    b_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, go_backwards=True)
 
     # answer part
     ag_emb = embedding(answer_good)
     ag_fl = f_lstm_attention(ag_emb)
     ag_bl = b_lstm_attention(ag_emb)
-    ag_out = merge([ag_fl, ag_bl], mode='concat', concat_axis=2)
-
-    ag_out_fl = f_lstm_3(ag_out)
-    ag_out_bl = b_lstm_3(ag_out)
-    ag_out = merge([ag_out_fl, ag_out_bl], mode='concat', concat_axis=2)
-
-    ag_out = Permute((2, 1))(ag_out)
-    ag_out = MaxPooling1D(2 * n_lstm_dims)(ag_out)
-    ag_out = Flatten()(ag_out)
+    ag_out = merge([ag_fl, ag_bl], mode='concat', concat_axis=1)
 
     ab_emb = embedding(answer_bad)
     ab_fl = f_lstm_attention(ab_emb)
     ab_bl = b_lstm_attention(ab_emb)
-    ab_out = merge([ab_fl, ab_bl], mode='concat', concat_axis=2)
-
-    ab_out_fl = f_lstm_3(ab_out)
-    ab_out_bl = b_lstm_3(ab_out)
-    ab_out = merge([ab_out_fl, ab_out_bl], mode='concat', concat_axis=2)
-
-    ab_out = Permute((2, 1))(ab_out)
-    ab_out = MaxPooling1D(2 * n_lstm_dims)(ab_out)
-    ab_out = Flatten()(ab_out)
+    ab_out = merge([ab_fl, ab_bl], mode='concat', concat_axis=1)
 
     # merge together
     # note: `cos` refers to "cosine similarity", i.e. similar vectors should go to 1
@@ -117,7 +84,7 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128):
     def loss(y_true, y_pred):
         return y_pred
 
-    # unfortunately, the hinge loss approach means the "accura cy" metric isn't very valuable
+    # unfortunately, the hinge loss approach means the "accuracy" metric isn't very valuable
     metrics = []
 
     train_model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
@@ -127,14 +94,13 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128):
 
 if __name__ == '__main__':
     # get the data set
-    maxlen = 200 # words
+    maxlen = 40 # words
 
     from utils.get_data import get_data_set, create_dictionary_from_qas
 
     dic = create_dictionary_from_qas()
     targets, questions, good_answers, bad_answers, n_dims = get_data_set(maxlen)
 
-    ### THIS MODEL PERFORMS WELL ON THE TEST SET
     train_model, test_model = make_model(maxlen, n_dims)
 
     print('Fitting model')