😛

attention_lstm.py

@@ -52,6 +52,8 @@ class AttentionLSTM(Recurrent):
         self.b_regularizer = regularizers.get(b_regularizer)
         self.dropout_W, self.dropout_U = dropout_W, dropout_U
 
+        self.attention_units = list()
+
         if self.dropout_W or self.dropout_U:
             self.uses_learning_phase = True
         super(AttentionLSTM, self).__init__(**kwargs)
@@ -181,7 +183,6 @@ class AttentionLSTM(Recurrent):
             return x
 
     def step(self, x, states):
-
         h_tm1 = states[0]
         c_tm1 = states[1]
         B_U = states[2]

insuranceqa.py

@@ -67,8 +67,8 @@ def get_eval(f_name):
         all_answers = [int(i) for i in g.strip().split(' ')]
 
         question = convert_from_idxs(q)
-        q_data.append(pad_sequences([question], maxlen=maxlen, padding='post', truncating='post', value=0))
-        a_data.append(pad_sequences([answers[i] for i in all_answers], maxlen=maxlen, padding='post', truncating='post', value=0))
+        q_data.append(pad_sequences([question], maxlen=maxlen_question, padding='post', truncating='post', value=0))
+        a_data.append(pad_sequences([answers[i] for i in all_answers], maxlen=maxlen_answer, padding='post', truncating='post', value=0))
         n_good.append(len(good_answers))
 
     return q_data, a_data, n_good
@@ -106,9 +106,9 @@ def get_data(f_name):
     random.shuffle(combined)
     q_data[:], ag_data[:], ab_data, targets[:] = zip(*combined)
 
-    q_data = pad_sequences(q_data, maxlen=maxlen, padding='post', truncating='post', value=0)
-    ag_data = pad_sequences(ag_data, maxlen=maxlen, padding='post', truncating='post', value=0)
-    ab_data = pad_sequences(ab_data, maxlen=maxlen, padding='post', truncating='post', value=0)
+    q_data = pad_sequences(q_data, maxlen=maxlen_question, padding='post', truncating='post', value=0)
+    ag_data = pad_sequences(ag_data, maxlen=maxlen_answer, padding='post', truncating='post', value=0)
+    ab_data = pad_sequences(ab_data, maxlen=maxlen_answer, padding='post', truncating='post', value=0)
     targets = np.asarray(targets)
 
     return q_data, ag_data, ab_data, targets
@@ -162,13 +162,14 @@ def get_mrr(model, questions, all_answers, n_good, n_eval=-1):
 
 # model parameters
 n_words = 22354
-maxlen = 40
+maxlen_question = 10
+maxlen_answer = 50
 
 # the model being used
 print('Generating model')
 
 from keras_attention_model import make_model
-train_model, test_model = make_model(maxlen, n_words, n_embed_dims=128, n_lstm_dims=256)
+train_model, test_model = make_model(maxlen_question, maxlen_answer, n_words, n_embed_dims=128, n_lstm_dims=256)
 
 print('Getting data')
 data_sets = [
@@ -179,16 +180,34 @@ data_sets = [
 q_data, ag_data, ab_data, targets = get_data(data_sets[0])
 qv_data, avg_data, avb_data, v_targets = get_data(data_sets[1])
 
+test_model.load_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5'))
+
 # found through experimentation that ~24 epochs generalized the best
 print('Fitting model')
-for i in range(100):
-    print(i)
+for i in range(30):
+    print('----- %d -----' % i)
     np.random.shuffle(ab_data)
     train_model.fit([q_data, ag_data, ab_data], targets, nb_epoch=1, batch_size=128, validation_data=[[qv_data, avg_data, avb_data], v_targets], shuffle=True)
 
+    if i % 100 == 0:
+        train_model.save_weights(os.path.join(models_path, 'iqa_model_for_training_iter_%d.h5' % i), overwrite=True)
+        test_model.save_weights(os.path.join(models_path, 'iqa_model_for_training_iter_%d.h5' % i), overwrite=True)
+
 train_model.save_weights(os.path.join(models_path, 'iqa_model_for_training.h5'), overwrite=True)
 test_model.save_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5'), overwrite=True)
 
+test_model.load_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5'))
+
+import keras.backend as K
+get_attention = K.function([test_model.layers[0].input, test_model.layers[1].input], [test_model.layers[3].get_output_at(0)])
+attention = get_attention([q_data[:20], ag_data[:20]])[0]
+
+for i in range(20):
+    print('----- %d -----' % i)
+    print(revert(q_data[i]))
+    print(revert(ag_data[i]))
+    print([np.linalg.norm(x) for x in attention[i]])
+
 # the model actually did really well, predicted correct vs. incorrect answer 85% of the time on the validation set
 test_model.load_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5'))
 print('Percent correct: {}'.format(get_accurate_percentage(test_model, q_data, ag_data, ab_data, n_eval='all')))

keras_attention_model.py

@@ -15,7 +15,7 @@ from word_embeddings import Word2VecEmbedding
 models_path = 'models/'
 
 
-def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128):
+def make_model(maxlen_question, maxlen_answer, n_words, n_lstm_dims=141, n_embed_dims=128):
     from keras.optimizers import RMSprop
 
     from attention_lstm import AttentionLSTM
@@ -25,39 +25,60 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128):
     import keras.backend as K
 
     # input
-    question = Input(shape=(maxlen,), dtype='int32')
-    answer_good = Input(shape=(maxlen,), dtype='int32')
-    answer_bad = Input(shape=(maxlen,), dtype='int32')
+    question = Input(shape=(maxlen_question,), dtype='int32', name='question_input')
+    answer_good = Input(shape=(maxlen_answer,), dtype='int32', name='answer_good_input')
+    answer_bad = Input(shape=(maxlen_answer,), dtype='int32', name='answer_bad_input')
 
     # language model
-    embedding = Embedding(n_words, n_embed_dims, mask_zero=True)
+    embedding = Embedding(n_words, n_embed_dims)
 
     # forward and backward lstms
-    f_lstm = LSTM(n_lstm_dims)
-    b_lstm = LSTM(n_lstm_dims, go_backwards=True)
-
-    # Note: Change concat_axis to 2 if return_sequences=True
+    f_lstm = LSTM(n_lstm_dims, name='fq_lstm', consume_less='mem', return_sequences=False)
+    b_lstm = LSTM(n_lstm_dims, name='bq_lstm', go_backwards=True, consume_less='mem', return_sequences=False)
 
     # question part
     q_emb = embedding(question)
+    q_emb = Convolution1D(nb_filter=64, filter_length=5)(q_emb)
+    q_emb = MaxPooling1D(pool_length=2)(q_emb)
     q_fl = f_lstm(q_emb)
     q_bl = b_lstm(q_emb)
-    q_out = merge([q_fl, q_bl], mode='concat', concat_axis=1)
+    q_out = merge([q_fl, q_bl], mode='concat', concat_axis=-1)
+    q_out = Dropout(0.25)(q_out)
+    # q_out = Permute((2, 1))(q_out)
+    # q_out = Convolution1D(nb_filter=64, filter_length=5)(q_out)
+    # q_out = MaxPooling1D(2)(q_out)
+    # q_out = Flatten()(q_out)
 
     # forward and backward attention lstms (paying attention to q_out)
-    f_lstm_attention = AttentionLSTM(n_lstm_dims, q_out)
-    b_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, go_backwards=True)
+    f_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, consume_less='mem', return_sequences=False)
+    b_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, go_backwards=True, consume_less='mem', return_sequences=False)
+
+    conv = Convolution1D(nb_filter=64, filter_length=5)
 
     # answer part
     ag_emb = embedding(answer_good)
+    ag_emb = conv(ag_emb)
+    ag_emb = MaxPooling1D(pool_length=2)(ag_emb)
     ag_fl = f_lstm_attention(ag_emb)
     ag_bl = b_lstm_attention(ag_emb)
-    ag_out = merge([ag_fl, ag_bl], mode='concat', concat_axis=1)
+    ag_out = merge([ag_fl, ag_bl], mode='concat', concat_axis=-1)
+    ag_out = Dropout(0.25)(ag_out)
+    # ag_out = Permute((2, 1))(ag_out)
+    # ag_out = conv(ag_out)
+    # ag_out = MaxPooling1D(2)(ag_out)
+    # ag_out = Flatten()(ag_out)
 
     ab_emb = embedding(answer_bad)
+    ab_emb = conv(ab_emb)
+    ab_emb = MaxPooling1D(pool_length=2)(ab_emb)
     ab_fl = f_lstm_attention(ab_emb)
     ab_bl = b_lstm_attention(ab_emb)
-    ab_out = merge([ab_fl, ab_bl], mode='concat', concat_axis=1)
+    ab_out = merge([ab_fl, ab_bl], mode='concat', concat_axis=-1)
+    ab_out = Dropout(0.25)(ab_out)
+    # ab_out = Permute((2, 1))(ab_out)
+    # ab_out = conv(ab_out)
+    # ab_out = MaxPooling1D(2)(ab_out)
+    # ab_out = Flatten()(ab_out)
 
     # merge together
     # note: `cos` refers to "cosine similarity", i.e. similar vectors should go to 1
@@ -65,7 +86,7 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128):
     good_out = merge([q_out, ag_out], name='good', mode='cos', dot_axes=1)
     bad_out = merge([q_out, ab_out], name='bad', mode='cos', dot_axes=1)
 
-    target = merge([good_out, bad_out], name='target', mode=lambda x: K.maximum(1e-3, 0.2 - x[0] + x[1]), output_shape=lambda x: x[0])
+    target = merge([good_out, bad_out], name='target', mode=lambda x: K.maximum(1e-3, 0.3 - x[0] + x[1]), output_shape=lambda x: x[0])
 
     train_model = Model(input=[question, answer_good, answer_bad], output=target)
     test_model = Model(input=[question, answer_good], output=good_out)