diff --git a/attention_lstm.py b/attention_lstm.py
index da958dd..de0a872 100644
--- a/attention_lstm.py
+++ b/attention_lstm.py
@@ -210,7 +210,7 @@ class AttentionLSTM(Recurrent):
         # Attention gate #
         ##################
 
-        m = self.activation(K.dot(h, self.U_a) + attention)
+        m = K.tanh(K.dot(h, self.U_a) + attention)
         # Intuitively it makes more sense to use a sigmoid (was getting some NaN problems
         # which I think might have been caused by the exponential function -> gradients blow up)
         # s = K.exp(K.dot(m, self.U_s))
diff --git a/insuranceqa.py b/insuranceqa.py
index d15d07b..e8bb58e 100644
--- a/insuranceqa.py
+++ b/insuranceqa.py
@@ -32,7 +32,7 @@ def to_idx(x):
 
 
 def convert_from_idxs(x):
-    return np.asarray([emb_d.get(idx_d[i], 22295) for i in x.strip().split(' ')])
+    return np.asarray([emb_d.get(idx_d[i], 0) for i in x.strip().split(' ')])
 
 
 def revert(x):
@@ -67,8 +67,8 @@ def get_eval(f_name):
         all_answers = [int(i) for i in g.strip().split(' ')]
 
         question = convert_from_idxs(q)
-        q_data.append(pad_sequences([question], maxlen=maxlen, padding='post', truncating='post', value=22295))
-        a_data.append(pad_sequences([answers[i] for i in all_answers], maxlen=maxlen, padding='post', truncating='post', value=22295))
+        q_data.append(pad_sequences([question], maxlen=maxlen, padding='post', truncating='post', value=0))
+        a_data.append(pad_sequences([answers[i] for i in all_answers], maxlen=maxlen, padding='post', truncating='post', value=0))
         n_good.append(len(good_answers))
 
     return q_data, a_data, n_good
@@ -106,9 +106,9 @@ def get_data(f_name):
     random.shuffle(combined)
     q_data[:], ag_data[:], ab_data, targets[:] = zip(*combined)
 
-    q_data = pad_sequences(q_data, maxlen=maxlen, padding='post', truncating='post', value=22295)
-    ag_data = pad_sequences(ag_data, maxlen=maxlen, padding='post', truncating='post', value=22295)
-    ab_data = pad_sequences(ab_data, maxlen=maxlen, padding='post', truncating='post', value=22295)
+    q_data = pad_sequences(q_data, maxlen=maxlen, padding='post', truncating='post', value=0)
+    ag_data = pad_sequences(ag_data, maxlen=maxlen, padding='post', truncating='post', value=0)
+    ab_data = pad_sequences(ab_data, maxlen=maxlen, padding='post', truncating='post', value=0)
     targets = np.asarray(targets)
 
     return q_data, ag_data, ab_data, targets
@@ -147,11 +147,17 @@ def get_mrr(model, questions, all_answers, n_good, n_eval=-1):
         sims = model.predict([qs, ans]).flatten()
         r = rankdata(sims)
 
-        x = 1 / float(max(r) - max(r[:n_good[i]]) + 1)
-        print(max(r) - max(r[:n_good[i]] + 1))
+        max_r = np.argmax(r)
+        max_n = np.argmax(r[:n_good[i]])
 
+        x = 1 / float(r[max_r] - r[max_n] + 1)
         c += x
 
+        print('---------- (%d)\nQuestion:' % i, revert(question[0]))
+        print('Desired answer:', revert(ans[max_n]))
+        print('Highest-rank answer:', revert(ans[max_r]))
+        print('Rank of best answer:', r[max_n])
+
     return c / len(questions)
 
 # model parameters
@@ -184,9 +190,9 @@ train_model.save_weights(os.path.join(models_path, 'iqa_model_for_training.h5'),
 test_model.save_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5'), overwrite=True)
 
 # the model actually did really well, predicted correct vs. incorrect answer 85% of the time on the validation set
-# test_model.load_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5'))
-# print('Percent correct: {}'.format(get_accurate_percentage(test_model, q_data, ag_data, ab_data, n_eval='all')))
+test_model.load_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5'))
+print('Percent correct: {}'.format(get_accurate_percentage(test_model, q_data, ag_data, ab_data, n_eval='all')))
 
 q_data, a_data, n_good = get_eval(data_sets[1])
-# test_model.load_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5'))
+test_model.load_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5'))
 print('MRR: {}'.format(get_mrr(test_model, q_data, a_data, n_good)))
diff --git a/keras_attention_model.py b/keras_attention_model.py
index 1e43ca6..e73aa53 100644
--- a/keras_attention_model.py
+++ b/keras_attention_model.py
@@ -7,7 +7,7 @@ import os
 
 from keras.engine import Merge
 from keras.layers import Lambda, MaxPooling1D, Dense, Flatten, Dropout, Masking, Embedding, TimeDistributed, \
-    Convolution1D
+    Convolution1D, Permute
 from keras.optimizers import SGD
 
 from word_embeddings import Word2VecEmbedding
@@ -37,6 +37,9 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128):
     f_lstm = LSTM(n_lstm_dims, return_sequences=True)
     b_lstm = LSTM(n_lstm_dims, go_backwards=True, return_sequences=True)
 
+    f_lstm_2 = LSTM(n_lstm_dims, return_sequences=True)
+    b_lstm_2 = LSTM(n_lstm_dims, go_backwards=True, return_sequences=True)
+
     # Note: Change concat_axis to 2 if return_sequences=True
 
     # question part
@@ -44,14 +47,22 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128):
     q_fl = f_lstm(q_emb)
     q_bl = b_lstm(q_emb)
     q_out = merge([q_fl, q_bl], mode='concat', concat_axis=2)
-    q_out = Convolution1D(64, 5)(q_out)
-    q_out = MaxPooling1D()(q_out)
+
+    q_out_fl = f_lstm_2(q_out)
+    q_out_bl = b_lstm_2(q_out)
+    q_out = merge([q_out_fl, q_out_bl], mode='concat', concat_axis=2)
+
+    q_out = Permute((2, 1))(q_out)
+    q_out = MaxPooling1D(2 * n_lstm_dims)(q_out)
     q_out = Flatten()(q_out)
 
     # forward and backward attention lstms (paying attention to q_out)
     f_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, return_sequences=True)
     b_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, go_backwards=True, return_sequences=True)
 
+    f_lstm_3 = LSTM(n_lstm_dims, return_sequences=True)
+    b_lstm_3 = LSTM(n_lstm_dims, go_backwards=True, return_sequences=True)
+
     conv = Convolution1D(64, 5)
 
     # answer part
@@ -59,16 +70,26 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128):
     ag_fl = f_lstm_attention(ag_emb)
     ag_bl = b_lstm_attention(ag_emb)
     ag_out = merge([ag_fl, ag_bl], mode='concat', concat_axis=2)
-    ag_out = conv(ag_out)
-    ag_out = MaxPooling1D()(ag_out)
+
+    ag_out_fl = f_lstm_3(ag_out)
+    ag_out_bl = b_lstm_3(ag_out)
+    ag_out = merge([ag_out_fl, ag_out_bl], mode='concat', concat_axis=2)
+
+    ag_out = Permute((2, 1))(ag_out)
+    ag_out = MaxPooling1D(2 * n_lstm_dims)(ag_out)
     ag_out = Flatten()(ag_out)
 
     ab_emb = embedding(answer_bad)
     ab_fl = f_lstm_attention(ab_emb)
     ab_bl = b_lstm_attention(ab_emb)
     ab_out = merge([ab_fl, ab_bl], mode='concat', concat_axis=2)
-    ab_out = conv(ab_out)
-    ab_out = MaxPooling1D()(ab_out)
+
+    ab_out_fl = f_lstm_3(ab_out)
+    ab_out_bl = b_lstm_3(ab_out)
+    ab_out = merge([ab_out_fl, ab_out_bl], mode='concat', concat_axis=2)
+
+    ab_out = Permute((2, 1))(ab_out)
+    ab_out = MaxPooling1D(2 * n_lstm_dims)(ab_out)
     ab_out = Flatten()(ab_out)
 
     # merge together
@@ -77,16 +98,15 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128):
     good_out = merge([q_out, ag_out], name='good', mode='cos', dot_axes=1)
     bad_out = merge([q_out, ab_out], name='bad', mode='cos', dot_axes=1)
 
-    target = merge([good_out, bad_out], name='target', mode=lambda x: K.maximum(0, 0.2 - x[0] + x[1]), output_shape=lambda x: x[0])
+    target = merge([good_out, bad_out], name='target', mode=lambda x: K.maximum(1e-3, 0.2 - x[0] + x[1]), output_shape=lambda x: x[0])
 
     train_model = Model(input=[question, answer_good, answer_bad], output=target)
     test_model = Model(input=[question, answer_good], output=good_out)
 
-    # need to choose binary crossentropy or mean squared error
     print('Compiling model...')
 
-    optimizer = RMSprop(lr=0.0001)
-    # optimizer = SGD(lr=0.001, momentum=0.9, nesterov=True)
+    optimizer = RMSprop(lr=0.0001, clipnorm=0.05)
+    # optimizer = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True, clipgrad=0.1)
 
     # this is more true to the paper: L = max{0, M - cosine(q, a+) + cosine(q, a-)}
     # below, "a" is a list of zeros and "b" is `target` above, i.e. 1 - cosine(q, a+) + cosine(q, a-)
@@ -107,7 +127,7 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128):
 
 if __name__ == '__main__':
     # get the data set
-    maxlen = 40 # words
+    maxlen = 200 # words
 
     from utils.get_data import get_data_set, create_dictionary_from_qas
 
@@ -118,5 +138,5 @@ if __name__ == '__main__':
     train_model, test_model = make_model(maxlen, n_dims)
 
     print('Fitting model')
-    train_model.fit([questions, good_answers, bad_answers], targets, nb_epoch=5, batch_size=32, validation_split=0.2)
+    train_model.fit([questions, good_answers, bad_answers], targets, nb_epoch=5, batch_size=128)
     train_model.save_weights(os.path.join(models_path, 'attention_lm_weights.h5'), overwrite=True)
diff --git a/results.notes b/results.notes
index c9d17eb..59a30f0 100644
--- a/results.notes
+++ b/results.notes
@@ -3,3 +3,25 @@ Single-layer bi-LSTM with max pooling, 40 words per sentence, loss margin of 0.2
  - Seemed to converge after about 20 epochs, with randomization between epochs
  - Using the pure embedding layer worked better than using the Word2Vec model (gave MRR ~0.09)
 
+CNN + ALSTM model seems to have good loss
+ - MRR ~0.33 (I think)
+
+CNN + ALSTM
+ - Example error:
+   - Question: be home insurance negotiable
+   - Desired answer: I not sure exactly what you mean but if you mean go without then no a mortgage company will not allow that and if you own outright it will be a foolish choice go without if you mean price negotiation (ranked 177)
+   - Highest-rank answer: no homeowner be not mandatory in Pennsylvania however if you be apply for any type of homeowner loan most bank will require you purchase insurance process the loan the bank will also ask be list as the lien holder if
+
+ - Example error:
+   - Question: what be Suze Orman advice on long term care insurance
+   - Desired answer: Suze Orman recommend long-term care insurance if it fit and 1 can medically qualify for it she be currently take care of her 90 year old mother who refuse to letSuze buy long-term care insurance on her when she can (rank ~460)
+   - Highest-rank answer: a good age buy long-term care insurance when you be in a good financial position and have some extra disposable income also if your health be good you will save thousand dollar potentially in premium need specific number how about
+
+ - Example error:
+   - Question: can you get Life Insurance on someone else
+   - Desired answer: you can get life insurance on someone else if you have an insurable interest with them an example of insurable interest be this : if a nonrelative owe you money you can take out a life insurance policy on them (rank 498)
+   - Highest-rank answer: I assume when you ask Doe someone have a life insurance policy on me , you be ask about someone other than your parent as an adult no one can take a policy out on you without you give your
+
+ - In general, the model predicts the topics well, but doesn't necessarily match it well with the question. This might be due to the masking issue.
+ - MRR: 0.30957
+
diff --git a/utils/dictionary.py b/utils/dictionary.py
index b8ec234..c37e418 100644
--- a/utils/dictionary.py
+++ b/utils/dictionary.py
@@ -11,9 +11,10 @@ from gensim.utils import tokenize
 
 
 class Dictionary:
-    def __init__(self):
+    def __init__(self, min_len=1):
         self._token_counts = dict()
-        self._id = 0
+        self._id = 1
+        self._min_len = min_len
 
         self.token2id = dict()
         self.id2token = list()
@@ -45,7 +46,7 @@ class Dictionary:
         return self.id2token[item] if 0 <= item < len(self.token2id) else 'UNKNOWN'
 
     def __len__(self):
-        return self._id + 2
+        return self._id + 1
 
     def convert(self, text):
         if isinstance(text, str):
diff --git a/utils/get_data.py b/utils/get_data.py
index 7f0a328..ae316cc 100644
--- a/utils/get_data.py
+++ b/utils/get_data.py
@@ -109,6 +109,9 @@ def get_data_set(maxlen, questions=None, answers=None, dic=None):
     for id, question in questions.items():
         qc = dic.convert(question['title'] + question['content'])[0]
 
+        if len(qc) < 5:
+            continue
+
         ggans = [dic.convert(a['answer'])[0] for a in answers[id] if int(a['score']) >= 3]
         bbans = [dic.convert(a['answer'])[0] for a in answers[id] if int(a['score']) < 3]
 
@@ -120,8 +123,8 @@ def get_data_set(maxlen, questions=None, answers=None, dic=None):
         targets += [0] * m
 
     targets = np.asarray(targets)
-    qs = pad_sequences(qs, maxlen=maxlen, padding='post', truncating='post', dtype='int32')
-    gans = pad_sequences(gans, maxlen=maxlen, padding='post', truncating='post', dtype='int32')
-    bans = pad_sequences(bans, maxlen=maxlen, padding='post', truncating='post', dtype='int32')
+    qs = pad_sequences(qs, maxlen=maxlen, padding='post', truncating='post', dtype='int32', value=0)
+    gans = pad_sequences(gans, maxlen=maxlen, padding='post', truncating='post', dtype='int32', value=0)
+    bans = pad_sequences(bans, maxlen=maxlen, padding='post', truncating='post', dtype='int32', value=0)
 
     return targets, qs, gans, bans, len(dic)