mirror of
https://github.com/wassname/keras-language-modeling.git
synced 2026-09-10 12:15:18 +08:00
switched around some stuff
This commit is contained in:
codekansas
+1
-1
@@ -210,7 +210,7 @@ class AttentionLSTM(Recurrent):
|
||||
# Attention gate #
|
||||
##################
|
||||
|
||||
m = self.activation(K.dot(h, self.U_a) + attention)
|
||||
m = K.tanh(K.dot(h, self.U_a) + attention)
|
||||
# Intuitively it makes more sense to use a sigmoid (was getting some NaN problems
|
||||
# which I think might have been caused by the exponential function -> gradients blow up)
|
||||
# s = K.exp(K.dot(m, self.U_s))
|
||||
|
||||
+17
-11
@@ -32,7 +32,7 @@ def to_idx(x):
|
||||
|
||||
|
||||
def convert_from_idxs(x):
|
||||
return np.asarray([emb_d.get(idx_d[i], 22295) for i in x.strip().split(' ')])
|
||||
return np.asarray([emb_d.get(idx_d[i], 0) for i in x.strip().split(' ')])
|
||||
|
||||
|
||||
def revert(x):
|
||||
@@ -67,8 +67,8 @@ def get_eval(f_name):
|
||||
all_answers = [int(i) for i in g.strip().split(' ')]
|
||||
|
||||
question = convert_from_idxs(q)
|
||||
q_data.append(pad_sequences([question], maxlen=maxlen, padding='post', truncating='post', value=22295))
|
||||
a_data.append(pad_sequences([answers[i] for i in all_answers], maxlen=maxlen, padding='post', truncating='post', value=22295))
|
||||
q_data.append(pad_sequences([question], maxlen=maxlen, padding='post', truncating='post', value=0))
|
||||
a_data.append(pad_sequences([answers[i] for i in all_answers], maxlen=maxlen, padding='post', truncating='post', value=0))
|
||||
n_good.append(len(good_answers))
|
||||
|
||||
return q_data, a_data, n_good
|
||||
@@ -106,9 +106,9 @@ def get_data(f_name):
|
||||
random.shuffle(combined)
|
||||
q_data[:], ag_data[:], ab_data, targets[:] = zip(*combined)
|
||||
|
||||
q_data = pad_sequences(q_data, maxlen=maxlen, padding='post', truncating='post', value=22295)
|
||||
ag_data = pad_sequences(ag_data, maxlen=maxlen, padding='post', truncating='post', value=22295)
|
||||
ab_data = pad_sequences(ab_data, maxlen=maxlen, padding='post', truncating='post', value=22295)
|
||||
q_data = pad_sequences(q_data, maxlen=maxlen, padding='post', truncating='post', value=0)
|
||||
ag_data = pad_sequences(ag_data, maxlen=maxlen, padding='post', truncating='post', value=0)
|
||||
ab_data = pad_sequences(ab_data, maxlen=maxlen, padding='post', truncating='post', value=0)
|
||||
targets = np.asarray(targets)
|
||||
|
||||
return q_data, ag_data, ab_data, targets
|
||||
@@ -147,11 +147,17 @@ def get_mrr(model, questions, all_answers, n_good, n_eval=-1):
|
||||
sims = model.predict([qs, ans]).flatten()
|
||||
r = rankdata(sims)
|
||||
|
||||
x = 1 / float(max(r) - max(r[:n_good[i]]) + 1)
|
||||
print(max(r) - max(r[:n_good[i]] + 1))
|
||||
max_r = np.argmax(r)
|
||||
max_n = np.argmax(r[:n_good[i]])
|
||||
|
||||
x = 1 / float(r[max_r] - r[max_n] + 1)
|
||||
c += x
|
||||
|
||||
print('---------- (%d)\nQuestion:' % i, revert(question[0]))
|
||||
print('Desired answer:', revert(ans[max_n]))
|
||||
print('Highest-rank answer:', revert(ans[max_r]))
|
||||
print('Rank of best answer:', r[max_n])
|
||||
|
||||
return c / len(questions)
|
||||
|
||||
# model parameters
|
||||
@@ -184,9 +190,9 @@ train_model.save_weights(os.path.join(models_path, 'iqa_model_for_training.h5'),
|
||||
test_model.save_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5'), overwrite=True)
|
||||
|
||||
# the model actually did really well, predicted correct vs. incorrect answer 85% of the time on the validation set
|
||||
# test_model.load_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5'))
|
||||
# print('Percent correct: {}'.format(get_accurate_percentage(test_model, q_data, ag_data, ab_data, n_eval='all')))
|
||||
test_model.load_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5'))
|
||||
print('Percent correct: {}'.format(get_accurate_percentage(test_model, q_data, ag_data, ab_data, n_eval='all')))
|
||||
|
||||
q_data, a_data, n_good = get_eval(data_sets[1])
|
||||
# test_model.load_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5'))
|
||||
test_model.load_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5'))
|
||||
print('MRR: {}'.format(get_mrr(test_model, q_data, a_data, n_good)))
|
||||
|
||||
+33
-13
@@ -7,7 +7,7 @@ import os
|
||||
|
||||
from keras.engine import Merge
|
||||
from keras.layers import Lambda, MaxPooling1D, Dense, Flatten, Dropout, Masking, Embedding, TimeDistributed, \
|
||||
Convolution1D
|
||||
Convolution1D, Permute
|
||||
from keras.optimizers import SGD
|
||||
|
||||
from word_embeddings import Word2VecEmbedding
|
||||
@@ -37,6 +37,9 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128):
|
||||
f_lstm = LSTM(n_lstm_dims, return_sequences=True)
|
||||
b_lstm = LSTM(n_lstm_dims, go_backwards=True, return_sequences=True)
|
||||
|
||||
f_lstm_2 = LSTM(n_lstm_dims, return_sequences=True)
|
||||
b_lstm_2 = LSTM(n_lstm_dims, go_backwards=True, return_sequences=True)
|
||||
|
||||
# Note: Change concat_axis to 2 if return_sequences=True
|
||||
|
||||
# question part
|
||||
@@ -44,14 +47,22 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128):
|
||||
q_fl = f_lstm(q_emb)
|
||||
q_bl = b_lstm(q_emb)
|
||||
q_out = merge([q_fl, q_bl], mode='concat', concat_axis=2)
|
||||
q_out = Convolution1D(64, 5)(q_out)
|
||||
q_out = MaxPooling1D()(q_out)
|
||||
|
||||
q_out_fl = f_lstm_2(q_out)
|
||||
q_out_bl = b_lstm_2(q_out)
|
||||
q_out = merge([q_out_fl, q_out_bl], mode='concat', concat_axis=2)
|
||||
|
||||
q_out = Permute((2, 1))(q_out)
|
||||
q_out = MaxPooling1D(2 * n_lstm_dims)(q_out)
|
||||
q_out = Flatten()(q_out)
|
||||
|
||||
# forward and backward attention lstms (paying attention to q_out)
|
||||
f_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, return_sequences=True)
|
||||
b_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, go_backwards=True, return_sequences=True)
|
||||
|
||||
f_lstm_3 = LSTM(n_lstm_dims, return_sequences=True)
|
||||
b_lstm_3 = LSTM(n_lstm_dims, go_backwards=True, return_sequences=True)
|
||||
|
||||
conv = Convolution1D(64, 5)
|
||||
|
||||
# answer part
|
||||
@@ -59,16 +70,26 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128):
|
||||
ag_fl = f_lstm_attention(ag_emb)
|
||||
ag_bl = b_lstm_attention(ag_emb)
|
||||
ag_out = merge([ag_fl, ag_bl], mode='concat', concat_axis=2)
|
||||
ag_out = conv(ag_out)
|
||||
ag_out = MaxPooling1D()(ag_out)
|
||||
|
||||
ag_out_fl = f_lstm_3(ag_out)
|
||||
ag_out_bl = b_lstm_3(ag_out)
|
||||
ag_out = merge([ag_out_fl, ag_out_bl], mode='concat', concat_axis=2)
|
||||
|
||||
ag_out = Permute((2, 1))(ag_out)
|
||||
ag_out = MaxPooling1D(2 * n_lstm_dims)(ag_out)
|
||||
ag_out = Flatten()(ag_out)
|
||||
|
||||
ab_emb = embedding(answer_bad)
|
||||
ab_fl = f_lstm_attention(ab_emb)
|
||||
ab_bl = b_lstm_attention(ab_emb)
|
||||
ab_out = merge([ab_fl, ab_bl], mode='concat', concat_axis=2)
|
||||
ab_out = conv(ab_out)
|
||||
ab_out = MaxPooling1D()(ab_out)
|
||||
|
||||
ab_out_fl = f_lstm_3(ab_out)
|
||||
ab_out_bl = b_lstm_3(ab_out)
|
||||
ab_out = merge([ab_out_fl, ab_out_bl], mode='concat', concat_axis=2)
|
||||
|
||||
ab_out = Permute((2, 1))(ab_out)
|
||||
ab_out = MaxPooling1D(2 * n_lstm_dims)(ab_out)
|
||||
ab_out = Flatten()(ab_out)
|
||||
|
||||
# merge together
|
||||
@@ -77,16 +98,15 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128):
|
||||
good_out = merge([q_out, ag_out], name='good', mode='cos', dot_axes=1)
|
||||
bad_out = merge([q_out, ab_out], name='bad', mode='cos', dot_axes=1)
|
||||
|
||||
target = merge([good_out, bad_out], name='target', mode=lambda x: K.maximum(0, 0.2 - x[0] + x[1]), output_shape=lambda x: x[0])
|
||||
target = merge([good_out, bad_out], name='target', mode=lambda x: K.maximum(1e-3, 0.2 - x[0] + x[1]), output_shape=lambda x: x[0])
|
||||
|
||||
train_model = Model(input=[question, answer_good, answer_bad], output=target)
|
||||
test_model = Model(input=[question, answer_good], output=good_out)
|
||||
|
||||
# need to choose binary crossentropy or mean squared error
|
||||
print('Compiling model...')
|
||||
|
||||
optimizer = RMSprop(lr=0.0001)
|
||||
# optimizer = SGD(lr=0.001, momentum=0.9, nesterov=True)
|
||||
optimizer = RMSprop(lr=0.0001, clipnorm=0.05)
|
||||
# optimizer = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True, clipgrad=0.1)
|
||||
|
||||
# this is more true to the paper: L = max{0, M - cosine(q, a+) + cosine(q, a-)}
|
||||
# below, "a" is a list of zeros and "b" is `target` above, i.e. 1 - cosine(q, a+) + cosine(q, a-)
|
||||
@@ -107,7 +127,7 @@ def make_model(maxlen, n_words, n_lstm_dims=141, n_embed_dims=128):
|
||||
|
||||
if __name__ == '__main__':
|
||||
# get the data set
|
||||
maxlen = 40 # words
|
||||
maxlen = 200 # words
|
||||
|
||||
from utils.get_data import get_data_set, create_dictionary_from_qas
|
||||
|
||||
@@ -118,5 +138,5 @@ if __name__ == '__main__':
|
||||
train_model, test_model = make_model(maxlen, n_dims)
|
||||
|
||||
print('Fitting model')
|
||||
train_model.fit([questions, good_answers, bad_answers], targets, nb_epoch=5, batch_size=32, validation_split=0.2)
|
||||
train_model.fit([questions, good_answers, bad_answers], targets, nb_epoch=5, batch_size=128)
|
||||
train_model.save_weights(os.path.join(models_path, 'attention_lm_weights.h5'), overwrite=True)
|
||||
|
||||
@@ -3,3 +3,25 @@ Single-layer bi-LSTM with max pooling, 40 words per sentence, loss margin of 0.2
|
||||
- Seemed to converge after about 20 epochs, with randomization between epochs
|
||||
- Using the pure embedding layer worked better than using the Word2Vec model (gave MRR ~0.09)
|
||||
|
||||
CNN + ALSTM model seems to have good loss
|
||||
- MRR ~0.33 (I think)
|
||||
|
||||
CNN + ALSTM
|
||||
- Example error:
|
||||
- Question: be home insurance negotiable
|
||||
- Desired answer: I not sure exactly what you mean but if you mean go without then no a mortgage company will not allow that and if you own outright it will be a foolish choice go without if you mean price negotiation (ranked 177)
|
||||
- Highest-rank answer: no homeowner be not mandatory in Pennsylvania however if you be apply for any type of homeowner loan most bank will require you purchase insurance process the loan the bank will also ask be list as the lien holder if
|
||||
|
||||
- Example error:
|
||||
- Question: what be Suze Orman advice on long term care insurance
|
||||
- Desired answer: Suze Orman recommend long-term care insurance if it fit and 1 can medically qualify for it she be currently take care of her 90 year old mother who refuse to letSuze buy long-term care insurance on her when she can (rank ~460)
|
||||
- Highest-rank answer: a good age buy long-term care insurance when you be in a good financial position and have some extra disposable income also if your health be good you will save thousand dollar potentially in premium need specific number how about
|
||||
|
||||
- Example error:
|
||||
- Question: can you get Life Insurance on someone else
|
||||
- Desired answer: you can get life insurance on someone else if you have an insurable interest with them an example of insurable interest be this : if a nonrelative owe you money you can take out a life insurance policy on them (rank 498)
|
||||
- Highest-rank answer: I assume when you ask Doe someone have a life insurance policy on me , you be ask about someone other than your parent as an adult no one can take a policy out on you without you give your
|
||||
|
||||
- In general, the model predicts the topics well, but doesn't necessarily match it well with the question. This might be due to the masking issue.
|
||||
- MRR: 0.30957
|
||||
|
||||
|
||||
+4
-3
@@ -11,9 +11,10 @@ from gensim.utils import tokenize
|
||||
|
||||
|
||||
class Dictionary:
|
||||
def __init__(self):
|
||||
def __init__(self, min_len=1):
|
||||
self._token_counts = dict()
|
||||
self._id = 0
|
||||
self._id = 1
|
||||
self._min_len = min_len
|
||||
|
||||
self.token2id = dict()
|
||||
self.id2token = list()
|
||||
@@ -45,7 +46,7 @@ class Dictionary:
|
||||
return self.id2token[item] if 0 <= item < len(self.token2id) else 'UNKNOWN'
|
||||
|
||||
def __len__(self):
|
||||
return self._id + 2
|
||||
return self._id + 1
|
||||
|
||||
def convert(self, text):
|
||||
if isinstance(text, str):
|
||||
|
||||
+6
-3
@@ -109,6 +109,9 @@ def get_data_set(maxlen, questions=None, answers=None, dic=None):
|
||||
for id, question in questions.items():
|
||||
qc = dic.convert(question['title'] + question['content'])[0]
|
||||
|
||||
if len(qc) < 5:
|
||||
continue
|
||||
|
||||
ggans = [dic.convert(a['answer'])[0] for a in answers[id] if int(a['score']) >= 3]
|
||||
bbans = [dic.convert(a['answer'])[0] for a in answers[id] if int(a['score']) < 3]
|
||||
|
||||
@@ -120,8 +123,8 @@ def get_data_set(maxlen, questions=None, answers=None, dic=None):
|
||||
targets += [0] * m
|
||||
|
||||
targets = np.asarray(targets)
|
||||
qs = pad_sequences(qs, maxlen=maxlen, padding='post', truncating='post', dtype='int32')
|
||||
gans = pad_sequences(gans, maxlen=maxlen, padding='post', truncating='post', dtype='int32')
|
||||
bans = pad_sequences(bans, maxlen=maxlen, padding='post', truncating='post', dtype='int32')
|
||||
qs = pad_sequences(qs, maxlen=maxlen, padding='post', truncating='post', dtype='int32', value=0)
|
||||
gans = pad_sequences(gans, maxlen=maxlen, padding='post', truncating='post', dtype='int32', value=0)
|
||||
bans = pad_sequences(bans, maxlen=maxlen, padding='post', truncating='post', dtype='int32', value=0)
|
||||
|
||||
return targets, qs, gans, bans, len(dic)
|
||||
|
||||
Reference in New Issue
Block a user