added answer_to_question generative model

answer_to_question.py

@@ -0,0 +1,130 @@
+from __future__ import print_function
+import numpy as np
+
+import os
+from keras.engine import Input
+from keras.layers import LSTM, RepeatVector, TimeDistributed, Dense, Activation
+from keras.models import Model
+
+# can remove this depending on ide...
+os.environ['INSURANCE_QA'] = '/media/moloch/HHD/MachineLearning/data/insuranceQA/pyenc'
+
+import sys
+
+try:
+    import cPickle as pickle
+except:
+    import pickle
+
+
+class InsuranceQA:
+    def __init__(self):
+        try:
+            data_path = os.environ['INSURANCE_QA']
+        except KeyError:
+            print("INSURANCE_QA is not set.  Set it to your clone of https://github.com/codekansas/insurance_qa_python")
+            sys.exit(1)
+        self.path = data_path
+        self.vocab = self.load('vocabulary')
+        self.table = InsuranceQA.VocabularyTable(self.vocab.values())
+
+    def load(self, name):
+        return pickle.load(open(os.path.join(self.path, name), 'rb'))
+
+    class VocabularyTable:
+        ''' Identical to CharacterTable from Keras example '''
+        def __init__(self, words):
+            self.words = sorted(set(words))
+            self.words_indices = dict((c, i) for i, c in enumerate(self.words))
+            self.indices_words = dict((i, c) for i, c in enumerate(self.words))
+
+        def encode(self, sentence, maxlen):
+            indices = np.zeros((maxlen, len(self.words)))
+            for i, w in enumerate(sentence):
+                if i == maxlen: break
+                indices[i, self.words_indices[w]] = 1
+            return indices
+
+        def decode(self, indices, calc_argmax=True):
+            if calc_argmax:
+                indices = indices.argmax(axis=-1)
+            return ' '.join(self.indices_words[x] for x in indices)
+
+def get_model(question_maxlen, answer_maxlen, vocab_len, n_hidden):
+    answer = Input(shape=(answer_maxlen, vocab_len))
+    # answer = Masking(mask_value=0.)(answer)
+
+    # encoder rnn
+    encode_rnn = LSTM(n_hidden, return_sequences=False)(answer)
+
+    # can add more layers
+    for i in range(2):
+        encode_rnn = LSTM(n_hidden, return_sequences=True)(encode_rnn)
+
+    # repeat it maxlen times
+    repeat_encoding = RepeatVector(question_maxlen)(encode_rnn)
+
+    # decoder rnn
+    decode_rnn = LSTM(n_hidden, return_sequences=True)(repeat_encoding)
+
+    # can add more layers
+    for i in range(2):
+        decode_rnn = LSTM(n_hidden, return_sequences=True)(decode_rnn)
+
+    # output
+    dense = TimeDistributed(Dense(vocab_len))(decode_rnn)
+    softmax = Activation('softmax')(dense)
+
+    # compile the model
+    model = Model([answer], [softmax])
+    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
+
+    return model
+
+if __name__ == '__main__':
+    question_maxlen, answer_maxlen = 10, 40
+
+    qa = InsuranceQA()
+    batch_size = 50
+    n_test = 10
+
+    print('Generating data...')
+    answers = qa.load('answers')
+    questions = qa.load('train')
+
+    def gen_questions(batch_size):
+        while True:
+            i = 0
+            question_idx = np.zeros(shape=(batch_size, question_maxlen, len(qa.vocab)))
+            answer_idx = np.zeros(shape=(batch_size, answer_maxlen, len(qa.vocab)))
+            for s in questions:
+                a = s['answers'][0]
+                answer = qa.table.encode([qa.vocab[x] for x in answers[a]], answer_maxlen)
+                question = qa.table.encode([qa.vocab[x] for x in s['question']], question_maxlen)
+                answer_idx[i] = answer
+                question_idx[i] = question
+                i += 1
+                if i == batch_size:
+                    yield ([answer_idx], [question_idx])
+                    i = 0
+
+    gen = gen_questions(batch_size)
+    test_gen = gen_questions(n_test)
+
+    print('Generating model...')
+    model = get_model(question_maxlen=question_maxlen, answer_maxlen=answer_maxlen,
+                      vocab_len=len(qa.vocab), n_hidden=128)
+
+    print('Training model...')
+    for iteration in range(1, 200):
+        print()
+        print('-' * 50)
+        print('Iteration', iteration)
+        model.fit_generator(gen, samples_per_epoch=100*batch_size, nb_epoch=10)
+
+        x, y = next(test_gen)
+        y = y[0]
+        pred = model.predict(x, verbose=0)
+        for i in range(n_test):
+            print('Expected: {}'.format(qa.table.decode(y[i])))
+            print('Predicted: {}'.format(qa.table.decode(pred[i])))

attention_lstm.py

@@ -1,12 +1,18 @@
 from __future__ import absolute_import
 
 from keras import backend as K
-from keras.layers import LSTM
+from keras.layers import LSTM, activations
 
 
 class AttentionLSTM(LSTM):
-    def __init__(self, output_dim, attention_vec, **kwargs):
+    def __init__(self, output_dim, attention_vec, attn_activation='tanh',
+                 attn_inner_activation='tanh', single_attn=False,
+                 n_attention_dim=None, **kwargs):
         self.attention_vec = attention_vec
+        self.attn_activation = activations.get(attn_activation)
+        self.attn_inner_activation = activations.get(attn_inner_activation)
+        self.single_attention_param = single_attn
+        self.n_attention_dim = output_dim if n_attention_dim is None else n_attention_dim
 
         super(AttentionLSTM, self).__init__(output_dim, **kwargs)
 
@@ -26,12 +32,14 @@ class AttentionLSTM(LSTM):
                                    name='{}_U_m'.format(self.name))
         self.b_m = K.zeros((self.output_dim,), name='{}_b_m'.format(self.name))
 
-        # self.U_s = self.inner_init((self.output_dim, self.output_dim),
-        #                            name='{}_U_s'.format(self.name))
-        # self.b_s = K.zeros((self.output_dim,), name='{}_b_s'.format(self.name))
-        self.U_s = self.inner_init((self.output_dim, 1),
-                                   name='{}_U_s'.format(self.name))
-        self.b_s = K.zeros((1,), name='{}_b_s'.format(self.name))
+        if self.single_attention_param:
+            self.U_s = self.inner_init((self.output_dim, 1),
+                                       name='{}_U_s'.format(self.name))
+            self.b_s = K.zeros((1,), name='{}_b_s'.format(self.name))
+        else:
+            self.U_s = self.inner_init((self.output_dim, self.output_dim),
+                                       name='{}_U_s'.format(self.name))
+            self.b_s = K.zeros((self.output_dim,), name='{}_b_s'.format(self.name))
 
         self.trainable_weights += [self.U_a, self.U_m, self.U_s, self.b_a, self.b_m, self.b_s]
 
@@ -43,13 +51,15 @@ class AttentionLSTM(LSTM):
         h, [h, c] = super(AttentionLSTM, self).step(x, states)
         attention = states[4]
 
-        m = K.tanh(K.dot(h, self.U_a) * attention + self.b_a)
+        m = self.attn_inner_activation(K.dot(h, self.U_a) * attention + self.b_a)
         # Intuitively it makes more sense to use a sigmoid (was getting some NaN problems
         # which I think might have been caused by the exponential function -> gradients blow up)
-        # s = K.exp(K.dot(m, self.U_s) + self.b_s)
-        s = K.tanh(K.dot(m, self.U_s) + self.b_s)
-        h = h * K.repeat_elements(s, self.output_dim, axis=1)
-        # h = h * s
+        s = self.attn_activation(K.dot(m, self.U_s) + self.b_s)
+
+        if self.single_attention_param:
+            h = h * K.repeat_elements(s, self.output_dim, axis=1)
+        else:
+            h = h * s
 
         return h, [h, c]
 

insurance_qa_embeddings.py

@@ -3,7 +3,6 @@ from __future__ import print_function
 import os
 import sys
 import random
-from time import strftime, gmtime
 
 import pickle
 

insurance_qa_eval.py

@@ -1,23 +1,30 @@
 from __future__ import print_function
 
 import os
+# can remove this depending on ide...
+os.environ['INSURANCE_QA'] = '/media/moloch/HHD/MachineLearning/data/insuranceQA/pyenc'
+
 import sys
 import random
 from time import strftime, gmtime
 
 import pickle
 
-from keras.optimizers import Adam, RMSprop
+from keras.optimizers import RMSprop
 from scipy.stats import rankdata
 
 from keras_models import *
 
 random.seed(42)
 
-
 class Evaluator:
-    def __init__(self, path, conf=None):
-        self.path = path
+    def __init__(self, conf=None):
+        try:
+            data_path = os.environ['INSURANCE_QA']
+        except KeyError:
+            print("INSURANCE_QA is not set.  Set it to your clone of https://github.com/codekansas/insurance_qa_python")
+            sys.exit(1)
+        self.path = data_path
         self.conf = dict() if conf is None else conf
         self.params = conf.get('training_params', dict())
         self.answers = self.load('answers')
@@ -107,6 +114,11 @@ class Evaluator:
             # random.shuffle(bad_answers)
             bad_answers = self.pada(random.sample(self.answers.values(), len(good_answers)))
 
+            # shuffle questions
+            zipped = zip(questions, good_answers)
+            random.shuffle(zipped)
+            questions[:], good_answers[:] = zip(*zipped)
+
             print('Epoch %d :: ' % (i+1), end='')
             self.print_time()
             model.fit([questions, good_answers, bad_answers], nb_epoch=1, batch_size=batch_size, validation_split=split)
@@ -200,30 +212,24 @@ class Evaluator:
         return top1s, mrrs
 
 if __name__ == '__main__':
-    try:
-        data_path = os.environ['INSURANCE_QA']
-    except KeyError:
-        print("INSURANCE_QA is not set.  Set it to your clone of https://github.com/codekansas/insurance_qa_python")
-        sys.exit(1)
-
     conf = {
-        'question_len': 20,
-        'answer_len': 100,
+        'question_len': 30,
+        'answer_len': 150,
         'n_words': 22353, # len(vocabulary) + 1
-        'margin': 0.02,
+        'margin': 0.2,
 
         'training_params': {
             'save_every': 1,
             'eval_every': 1,
             'batch_size': 128,
             'nb_epoch': 1000,
-            'validation_split': 0.2,
+            'validation_split': 0.1,
             'optimizer': RMSprop(clip_norm=0.1), # Adam(clip_norm=0.1),
             'n_eval': 20,
 
             'evaluate_all_threshold': {
                 'mode': 'all',
-                'top1': 0.5,
+                'top1': 0.4,
             },
         },
 
@@ -247,7 +253,7 @@ if __name__ == '__main__':
         }
     }
 
-    evaluator = Evaluator(data_path, conf)
+    evaluator = Evaluator(conf)
 
     ##### Define model ######
     model = AttentionModel(conf)
@@ -269,7 +275,7 @@ if __name__ == '__main__':
     language_model.layers[2].set_weights([weights])
 
     # train the model
-    # evaluator.load_epoch(model, 25)
+    # evaluator.load_epoch(model, 225)
     evaluator.train(model)
 
     # evaluate mrr for a particular epoch

keras_models.py

@@ -4,7 +4,7 @@ from abc import abstractmethod
 
 from keras.engine import Input
 from keras.layers import merge, Embedding, Dropout, Convolution1D, Lambda, Activation, LSTM, Dense, TimeDistributed, \
-    ActivityRegularization, Flatten
+    ActivityRegularization
 from keras import backend as K
 from keras.models import Model
 
@@ -240,7 +240,7 @@ class AttentionModel(LanguageModel):
         answer = self.get_answer()
 
         # add embedding layers
-        embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 100))
+        embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 100), mask_zero=False)
         question_embedding = embedding(question)
         answer_embedding = embedding(answer)
 
@@ -256,28 +256,23 @@ class AttentionModel(LanguageModel):
         # question rnn part
         f_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True)
         b_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True, go_backwards=True)
-        question_rnn = merge([f_rnn(question_dropout), b_rnn(question_dropout)], mode='concat', concat_axis=-1)
-        question_dropout = dropout(question_rnn)
-
-        # regularize
-        regularize = ActivityRegularization(l2=0.0001)
-        question_dropout = regularize(question_dropout)
-
-        # could add convolution layer here (as in paper)
+        question_f_rnn = f_rnn(question_dropout)
+        question_b_rnn = b_rnn(question_dropout)
+        question_f_dropout = dropout(question_f_rnn)
+        question_b_dropout = dropout(question_b_rnn)
 
         # maxpooling
         maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]))
-        question_pool = maxpool(question_dropout)
+        question_pool = merge([maxpool(question_f_dropout), maxpool(question_b_dropout)], mode='concat', concat_axis=-1)
 
         # answer rnn part
-        f_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, return_sequences=True)
-        b_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, return_sequences=True, go_backwards=True)
-        # f_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True)
-        # b_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True, go_backwards=True)
-        answer_rnn = merge([f_rnn(answer_dropout), b_rnn(answer_dropout)], mode='concat', concat_axis=-1)
-        answer_dropout = dropout(answer_rnn)
-        answer_dropout = regularize(answer_dropout)
-        answer_pool = maxpool(answer_dropout)
+        f_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, single_attn=True, return_sequences=True)
+        b_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, single_attn=True, return_sequences=True, go_backwards=True)
+        answer_f_rnn = f_rnn(answer_dropout)
+        answer_b_rnn = b_rnn(answer_dropout)
+        answer_f_dropout = dropout(answer_f_rnn)
+        answer_b_dropout = dropout(answer_b_rnn)
+        answer_pool = merge([maxpool(answer_f_dropout), maxpool(answer_b_dropout)], mode='concat', concat_axis=-1)
 
         # activation
         activation = Activation('tanh')