changed testing part

seq2seq/answer_to_question.py

@@ -0,0 +1,157 @@
+'''
+The training model learns to generate a "question" which contains all the same
+words as the original question. So it isn't really learning a sequence, but the
+result is interesting.
+'''
+
+from __future__ import print_function
+import numpy as np
+
+import os
+from keras.engine import Input
+from keras.layers import LSTM, RepeatVector, TimeDistributed, Dense, Activation, Masking, merge, activations, Lambda, \
+    ActivityRegularization
+from keras.models import Model
+import keras.backend as K
+
+# can remove this depending on ide...
+os.environ['INSURANCE_QA'] = '/media/moloch/HHD/MachineLearning/data/insuranceQA/pyenc'
+
+import sys
+
+try:
+    import cPickle as pickle
+except:
+    import pickle
+
+
+class InsuranceQA:
+    def __init__(self):
+        try:
+            data_path = os.environ['INSURANCE_QA']
+        except KeyError:
+            print("INSURANCE_QA is not set.  Set it to your clone of https://github.com/codekansas/insurance_qa_python")
+            sys.exit(1)
+        self.path = data_path
+        self.vocab = self.load('vocabulary')
+        self.table = InsuranceQA.VocabularyTable(self.vocab.values())
+
+    def load(self, name):
+        return pickle.load(open(os.path.join(self.path, name), 'rb'))
+
+    class VocabularyTable:
+        ''' Identical to CharacterTable from Keras example '''
+        def __init__(self, words):
+            self.words = sorted(set(words))
+            self.words_indices = dict((c, i) for i, c in enumerate(self.words))
+            self.indices_words = dict((i, c) for i, c in enumerate(self.words))
+
+        def encode(self, sentence, maxlen):
+            indices = np.zeros((maxlen, len(self.words)))
+            for i, w in enumerate(sentence):
+                if i == maxlen: break
+                indices[i, self.words_indices[w]] = 1
+            return indices
+
+        def decode(self, indices, calc_argmax=True, noise=0.2):
+            if calc_argmax:
+                indices = [self.sample(i, noise=noise) for i in indices]
+            return ' '.join(self.indices_words[x] for x in indices)
+
+        def sample(self, index, noise=0.2):
+            index = np.log(index) / noise
+            index = np.exp(index) / np.sum(np.exp(index))
+            index = np.argmax(np.random.multinomial(1, index, 1))
+            return index
+
+def get_model(question_maxlen, answer_maxlen, vocab_len, n_hidden):
+    answer = Input(shape=(answer_maxlen, vocab_len))
+    masked = Masking(mask_value=0.)(answer)
+
+    # encoder rnn
+    encode_rnn = LSTM(n_hidden, return_sequences=True, dropout_U=0.2)(masked)
+    encode_rnn = LSTM(n_hidden, return_sequences=False, dropout_U=0.2)(encode_rnn)
+
+    encode_brnn = LSTM(n_hidden, return_sequences=True, go_backwards=True, dropout_U=0.2)(masked)
+    encode_brnn = LSTM(n_hidden, return_sequences=False, go_backwards=True, dropout_U=0.2)(encode_brnn)
+
+    # repeat it maxlen times
+    repeat_encoding_rnn = RepeatVector(question_maxlen)(encode_rnn)
+    repeat_encoding_brnn = RepeatVector(question_maxlen)(encode_brnn)
+
+    # decoder rnn
+    decode_rnn = LSTM(n_hidden, return_sequences=True, dropout_U=0.2, dropout_W=0.5)(repeat_encoding_rnn)
+    decode_rnn = LSTM(n_hidden, return_sequences=True, dropout_U=0.2)(decode_rnn)
+
+    decode_brnn = LSTM(n_hidden, return_sequences=True, go_backwards=True, dropout_U=0.2, dropout_W=0.5)(repeat_encoding_brnn)
+    decode_brnn = LSTM(n_hidden, return_sequences=True, go_backwards=True, dropout_U=0.2)(decode_brnn)
+
+    merged_output = merge([decode_rnn, decode_brnn], mode='concat', concat_axis=-1)
+
+    # output
+    dense = TimeDistributed(Dense(vocab_len))(merged_output)
+    regularized = ActivityRegularization(l2=1)(dense)
+    softmax = Activation('softmax')(regularized)
+
+    # compile the prediction model
+    model = Model([answer], [softmax])
+    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
+
+    return model
+
+if __name__ == '__main__':
+    question_maxlen, answer_maxlen = 20, 60
+
+    qa = InsuranceQA()
+    batch_size = 50
+    n_test = 5
+
+    print('Generating data...')
+    answers = qa.load('answers')
+
+    def gen_questions(batch_size, test=False):
+        if test:
+            questions = qa.load('test1')
+        else:
+            questions = qa.load('train')
+        while True:
+            i = 0
+            question_idx = np.zeros(shape=(batch_size, question_maxlen, len(qa.vocab)))
+            answer_idx = np.zeros(shape=(batch_size, answer_maxlen, len(qa.vocab)))
+            for s in questions:
+                if test:
+                    ans = s['good']
+                else:
+                    ans = s['answers']
+                for a in ans:
+                    answer = qa.table.encode([qa.vocab[x] for x in answers[a]], answer_maxlen)
+                    question = qa.table.encode([qa.vocab[x] for x in s['question']], question_maxlen)
+                    question = np.amax(question, axis=0, keepdims=False)
+                    answer_idx[i] = answer
+                    question_idx[i] = question
+                    i += 1
+                    if i == batch_size:
+                        yield ([answer_idx], [question_idx])
+                        i = 0
+
+    gen = gen_questions(batch_size)
+    test_gen = gen_questions(n_test, test=True)
+
+    print('Generating model...')
+    model = get_model(question_maxlen=question_maxlen, answer_maxlen=answer_maxlen, vocab_len=len(qa.vocab), n_hidden=128)
+
+    print('Training model...')
+    for iteration in range(1, 200):
+        print()
+        print('-' * 50)
+        print('Iteration', iteration)
+        model.fit_generator(gen, samples_per_epoch=100*batch_size, nb_epoch=10)
+
+        x, y = next(test_gen)
+        y = y[0]
+        pred = model.predict(x, verbose=0)
+        for noise in [0.2, 0.5, 1.0, 1.2]: # not sure what noise values would be good
+            print(' Noise: {}'.format(noise))
+            for i in range(n_test):
+                print('    Expected: {}'.format(qa.table.decode(y[i])))
+                print('    Predicted: {}'.format(qa.table.decode(pred[i], noise=noise)))

seq2seq/seq2seq.py

@@ -0,0 +1,30 @@
+import keras.backend as K
+from keras.engine import Input
+from keras.layers import LSTM, TimeDistributed, Dense
+
+
+class Seq2Seq:
+    def __init__(self, encode_seq_length, decode_seq_length, n_symbols, **params):
+        self.input = Input(shape=(encode_seq_length, n_symbols,))
+
+        self.encode_seq_length = encode_seq_length
+        self.decode_seq_length = decode_seq_length
+        self.n_symbols = n_symbols
+        self.params = params
+
+        encoder = self.build_encoder(self.input)
+        decoder = self.build_decoder(encoder)
+
+    def get_param(self, param, default):
+        if param in self.params:
+            return self.params[param]
+        print('Could not find param "{}" in params: Using default value {}'.format(param, default))
+        return default
+
+    def build_encoder(self, input):
+        lstm = LSTM(self.get_param('n_lstm_dims', 100), return_sequences=False)(input)
+        dense = Dense(self.n_symbols)(lstm)
+        return dense
+
+    def build_decoder(self, input):
+        pass