From dfc585c2657f787b795aed1f5209307618748aad Mon Sep 17 00:00:00 2001 From: codekansas Date: Tue, 19 Apr 2016 10:06:19 -0400 Subject: [PATCH] tested using word2vec embeddings :-1: --- insuranceqa.py | 2 ++ keras_attention_model.py | 12 +++++++++--- results.notes | 1 + 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/insuranceqa.py b/insuranceqa.py index 08b8bf6..d15d07b 100644 --- a/insuranceqa.py +++ b/insuranceqa.py @@ -148,6 +148,7 @@ def get_mrr(model, questions, all_answers, n_good, n_eval=-1): r = rankdata(sims) x = 1 / float(max(r) - max(r[:n_good[i]]) + 1) + print(max(r) - max(r[:n_good[i]] + 1)) c += x @@ -175,6 +176,7 @@ qv_data, avg_data, avb_data, v_targets = get_data(data_sets[1]) # found through experimentation that ~24 epochs generalized the best print('Fitting model') for i in range(100): + print(i) np.random.shuffle(ab_data) train_model.fit([q_data, ag_data, ab_data], targets, nb_epoch=1, batch_size=128, validation_data=[[qv_data, avg_data, avb_data], v_targets], shuffle=True) diff --git a/keras_attention_model.py b/keras_attention_model.py index 2b9572d..1e43ca6 100644 --- a/keras_attention_model.py +++ b/keras_attention_model.py @@ -6,7 +6,8 @@ from __future__ import print_function import os from keras.engine import Merge -from keras.layers import Lambda, MaxPooling1D, Dense, Flatten, Dropout, Masking, Embedding, TimeDistributed +from keras.layers import Lambda, MaxPooling1D, Dense, Flatten, Dropout, Masking, Embedding, TimeDistributed, \ + Convolution1D from keras.optimizers import SGD from word_embeddings import Word2VecEmbedding @@ -29,8 +30,8 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128): answer_bad = Input(shape=(maxlen,), dtype='int32') # language model - # embedding = Embedding(n_words, n_embed_dims) - embedding = Word2VecEmbedding(os.path.join(models_path, 'word2vec.model')) + embedding = Embedding(n_words, n_embed_dims) + # embedding = Word2VecEmbedding(os.path.join(models_path, 'word2vec.model')) # forward and backward lstms f_lstm = LSTM(n_lstm_dims, return_sequences=True) @@ -43,6 +44,7 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128): q_fl = f_lstm(q_emb) q_bl = b_lstm(q_emb) q_out = merge([q_fl, q_bl], mode='concat', concat_axis=2) + q_out = Convolution1D(64, 5)(q_out) q_out = MaxPooling1D()(q_out) q_out = Flatten()(q_out) @@ -50,11 +52,14 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128): f_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, return_sequences=True) b_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, go_backwards=True, return_sequences=True) + conv = Convolution1D(64, 5) + # answer part ag_emb = embedding(answer_good) ag_fl = f_lstm_attention(ag_emb) ag_bl = b_lstm_attention(ag_emb) ag_out = merge([ag_fl, ag_bl], mode='concat', concat_axis=2) + ag_out = conv(ag_out) ag_out = MaxPooling1D()(ag_out) ag_out = Flatten()(ag_out) @@ -62,6 +67,7 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128): ab_fl = f_lstm_attention(ab_emb) ab_bl = b_lstm_attention(ab_emb) ab_out = merge([ab_fl, ab_bl], mode='concat', concat_axis=2) + ab_out = conv(ab_out) ab_out = MaxPooling1D()(ab_out) ab_out = Flatten()(ab_out) diff --git a/results.notes b/results.notes index e5eedf1..c9d17eb 100644 --- a/results.notes +++ b/results.notes @@ -1,4 +1,5 @@ Single-layer bi-LSTM with max pooling, 40 words per sentence, loss margin of 0.2 - MRR ~0.17 - Seemed to converge after about 20 epochs, with randomization between epochs + - Using the pure embedding layer worked better than using the Word2Vec model (gave MRR ~0.09)