wassname/keras-language-modeling
1
0
Fork 0
mirror of https://github.com/wassname/keras-language-modeling.git synced 2026-09-09 11:25:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

changed testing part

Browse Source
This commit is contained in:
codekansas
2016-05-07 13:33:23 -04:00
parent e61141e892
commit dd1960e2f9
5 changed files with 153 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
insurance_qa_eval.py
+41 -32
View File
@@ -10,7 +10,7 @@ from time import strftime, gmtime
import pickle
from keras.optimizers import RMSprop
from keras.optimizers import RMSprop, Adam
from scipy.stats import rankdata
from keras_models import *
@@ -108,26 +108,32 @@ class Evaluator:
good_answers = self.pada(good_answers)
# bad_answers = self.pada(random.sample(self.answers.values(), len(good_answers)))
for i in range(nb_epoch):
val_loss = {'loss': 1., 'epoch': 0}
for i in range(1, nb_epoch):
# bad_answers = np.roll(good_answers, random.randint(10, len(questions) - 10))
# bad_answers = good_answers.copy()
# random.shuffle(bad_answers)
bad_answers = self.pada(random.sample(self.answers.values(), len(good_answers)))
# shuffle questions
zipped = zip(questions, good_answers)
random.shuffle(zipped)
questions[:], good_answers[:] = zip(*zipped)
# zipped = zip(questions, good_answers)
# random.shuffle(zipped)
# questions[:], good_answers[:] = zip(*zipped)
print('Epoch %d :: ' % (i+1), end='')
print('Epoch %d :: ' % i, end='')
self.print_time()
model.fit([questions, good_answers, bad_answers], nb_epoch=1, batch_size=batch_size, validation_split=split)
hist = model.fit([questions, good_answers, bad_answers], nb_epoch=1, batch_size=batch_size, validation_split=split)
if eval_every is not None and (i+1) % eval_every == 0:
if hist.history['val_loss'][0] < val_loss['loss']:
val_loss = {'loss': hist.history['val_loss'][0], 'epoch': i}
print('Best: Loss = {}, Epoch = {}'.format(val_loss['loss'], val_loss['epoch']))
if eval_every is not None and i % eval_every == 0:
self.get_mrr(model)
if save_every is not None and (i+1) % save_every == 0:
self.save_epoch(model, (i+1))
if save_every is not None and i % save_every == 0:
self.save_epoch(model, i)
##### Evaluation #####
@@ -160,13 +166,13 @@ class Evaluator:
c_1, c_2 = 0, 0
c = 0
for i, d in enumerate(data):
if evaluate_all:
self.prog_bar(i, len(data))
answers = self.pada([self.answers[i] for i in d['good'] + d['bad']])
question = self.padq([d['question']] * len(d['good'] + d['bad']))
indices = d['good'] + d['bad']
answers = self.pada([self.answers[i] for i in indices])
question = self.padq([d['question']] * len(indices))
n_good = len(d['good'])
sims = model.predict([question, answers], batch_size=500).flatten()
@@ -175,6 +181,10 @@ class Evaluator:
max_r = np.argmax(r)
max_n = np.argmax(r[:n_good])
# print(' '.join(self.revert(d['question'])))
# print(' '.join(self.revert(self.answers[indices[max_r]])))
# print(' '.join(self.revert(self.answers[indices[max_n]])))
c_1 += 1 if max_r == max_n else 0
c_2 += 1 / float(r[max_r] - r[max_n] + 1)
@@ -204,7 +214,7 @@ class Evaluator:
evaluate_all = evaluate_all or any([x >= mrr_theshold for x in mrrs])
else:
evaluate_all = evaluate_all or all([x >= top1_threshold for x in top1s])
evaluate_all = evaluate_all and all([x >= mrr_theshold for x in mrrs])
evaluate_all = evaluate_all or all([x >= mrr_theshold for x in mrrs])
if evaluate_all:
return self.get_mrr(model, evaluate_all=True)
@@ -213,19 +223,20 @@ class Evaluator:
if __name__ == '__main__':
conf = {
'question_len': 30,
'answer_len': 150,
'question_len': 100,
'answer_len': 100,
'n_words': 22353, # len(vocabulary) + 1
'margin': 0.2,
'margin': 0.009,
'training_params': {
'save_every': 1,
'eval_every': 1,
# 'eval_every': 1,
'batch_size': 128,
'nb_epoch': 1000,
'validation_split': 0.1,
'optimizer': RMSprop(clip_norm=0.1), # Adam(clip_norm=0.1),
'n_eval': 20,
'validation_split': 0.2,
'optimizer': 'adam',
# 'optimizer': Adam(clip_norm=0.1),
# 'n_eval': 100,
'evaluate_all_threshold': {
'mode': 'all',
@@ -238,15 +249,17 @@ if __name__ == '__main__':
'n_hidden': 200,
# convolution
'nb_filters': 1000,
'nb_filters': 1000, # * 4
'conv_activation': 'relu',
# recurrent
'n_lstm_dims': 141,
'initial_embed_weights': np.load('word2vec_100_dim.embeddings'),
},
'similarity_params': {
'mode': 'gesd',
'mode': 'cosine',
'gamma': 1,
'c': 1,
'd': 2,
@@ -256,28 +269,24 @@ if __name__ == '__main__':
evaluator = Evaluator(conf)
##### Define model ######
model = AttentionModel(conf)
model = ConvolutionModel(conf)
optimizer = conf.get('training_params', dict()).get('optimizer', 'adam')
model.compile(optimizer=optimizer)
import numpy as np
# save embedding layer
# evaluator.load_epoch(model, 33)
# embedding_layer = model.prediction_model.layers[2].layers[2]
# evaluator.load_epoch(model, 100)
# evaluator.train(model)
# weights = embedding_layer.get_weights()[0]
# np.save(open('models/embedding_200_dim.h5', 'wb'), weights)
# load pre-trained embedding layer
weights = np.load('word2vec_100_dim.embeddings')
language_model = model.prediction_model.layers[2]
language_model.layers[2].set_weights([weights])
# np.save(open('models/embedding_1000_dim.h5', 'wb'), weights)
# train the model
# evaluator.load_epoch(model, 225)
# evaluator.load_epoch(model, 6)
evaluator.train(model)
# evaluate mrr for a particular epoch
# evaluator.load_epoch(model, 53)
# evaluator.load_epoch(model, 22)
# evaluator.get_mrr(model, evaluate_all=True)
keras_models.py
+29 -14
View File
@@ -129,7 +129,7 @@ class LanguageModel:
def fit(self, x, **kwargs):
assert self.training_model is not None, 'Must compile the model before fitting data'
y = np.zeros(shape=x[0].shape[:1])
self.training_model.fit(x, y, **kwargs)
return self.training_model.fit(x, y, **kwargs)
def predict(self, x, **kwargs):
return self.prediction_model.predict(x, **kwargs)
@@ -149,7 +149,12 @@ class EmbeddingModel(LanguageModel):
answer = self.get_answer()
# add embedding layers
embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 141))
weights = self.model_params.get('initial_embed_weights', None)
weights = weights if weights is None else [weights]
embedding = Embedding(input_dim=self.config['n_words'],
output_dim=self.model_params.get('n_embed_dims', 100),
weights=weights,
mask_zero=True)
question_embedding = embedding(question)
answer_embedding = embedding(answer)
@@ -172,6 +177,8 @@ class EmbeddingModel(LanguageModel):
class ConvolutionModel(LanguageModel):
### Validation loss at Epoch 65: 2.4e-6
def build(self):
assert self.config['question_len'] == self.config['answer_len']
@@ -179,13 +186,17 @@ class ConvolutionModel(LanguageModel):
answer = self.get_answer()
# add embedding layers
embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 100))
weights = self.model_params.get('initial_embed_weights', None)
weights = weights if weights is None else [weights]
embedding = Embedding(input_dim=self.config['n_words'],
output_dim=self.model_params.get('n_embed_dims', 100),
weights=weights)
question_embedding = embedding(question)
answer_embedding = embedding(answer)
# turn off layer updating
embedding.params = []
embedding.updates = []
# embedding.params = []
# embedding.updates = []
# dropout
dropout = Dropout(0.25)
@@ -213,10 +224,6 @@ class ConvolutionModel(LanguageModel):
question_cnn = merge([cnn(question_dropout) for cnn in cnns], mode='concat')
answer_cnn = merge([cnn(answer_dropout) for cnn in cnns], mode='concat')
# regularization
question_cnn = ActivityRegularization(l2=0.0001)(question_cnn)
answer_cnn = ActivityRegularization(l2=0.0001)(answer_cnn)
# dropout
question_dropout = dropout(question_cnn)
answer_dropout = dropout(answer_cnn)
@@ -240,7 +247,12 @@ class AttentionModel(LanguageModel):
answer = self.get_answer()
# add embedding layers
embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 100), mask_zero=False)
weights = self.model_params.get('initial_embed_weights', None)
weights = weights if weights is None else [weights]
embedding = Embedding(input_dim=self.config['n_words'],
output_dim=self.model_params.get('n_embed_dims', 100),
weights=weights,
mask_zero=True)
question_embedding = embedding(question)
answer_embedding = embedding(answer)
@@ -254,8 +266,9 @@ class AttentionModel(LanguageModel):
answer_dropout = dropout(answer_embedding)
# question rnn part
f_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True)
b_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True, go_backwards=True)
f_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True, consume_less='mem')
b_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True, consume_less='mem',
go_backwards=True)
question_f_rnn = f_rnn(question_dropout)
question_b_rnn = b_rnn(question_dropout)
question_f_dropout = dropout(question_f_rnn)
@@ -266,8 +279,10 @@ class AttentionModel(LanguageModel):
question_pool = merge([maxpool(question_f_dropout), maxpool(question_b_dropout)], mode='concat', concat_axis=-1)
# answer rnn part
f_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, single_attn=True, return_sequences=True)
b_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, single_attn=True, return_sequences=True, go_backwards=True)
f_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, single_attn=True,
return_sequences=True, consume_less='mem')
b_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, single_attn=True,
return_sequences=True, consume_less='mem', go_backwards=True)
answer_f_rnn = f_rnn(answer_dropout)
answer_b_rnn = b_rnn(answer_dropout)
answer_f_dropout = dropout(answer_f_rnn)
seq2seq/__init__.py
View File
answer_to_question.py → seq2seq/answer_to_question.py
+53 -31
View File
@@ -1,10 +1,18 @@
'''
The training model learns to generate a "question" which contains all the same
words as the original question. So it isn't really learning a sequence, but the
result is interesting.
'''
from __future__ import print_function
import numpy as np
import os
from keras.engine import Input
from keras.layers import LSTM, RepeatVector, TimeDistributed, Dense, Activation, Masking, merge
from keras.layers import LSTM, RepeatVector, TimeDistributed, Dense, Activation, Masking, merge, activations, Lambda, \
ActivityRegularization
from keras.models import Model
import keras.backend as K
# can remove this depending on ide...
os.environ['INSURANCE_QA'] = '/media/moloch/HHD/MachineLearning/data/insuranceQA/pyenc'
@@ -45,47 +53,54 @@ class InsuranceQA:
indices[i, self.words_indices[w]] = 1
return indices
def decode(self, indices, calc_argmax=True, noise=0):
def decode(self, indices, calc_argmax=True, noise=0.2):
if calc_argmax:
indices = indices + np.random.rand(*indices.shape) * noise
indices = indices.argmax(axis=-1)
indices = [self.sample(i, noise=noise) for i in indices]
return ' '.join(self.indices_words[x] for x in indices)
def sample(self, index, noise=0.2):
index = np.log(index) / noise
index = np.exp(index) / np.sum(np.exp(index))
index = np.argmax(np.random.multinomial(1, index, 1))
return index
def get_model(question_maxlen, answer_maxlen, vocab_len, n_hidden):
answer = Input(shape=(answer_maxlen, vocab_len))
masked = Masking(mask_value=0.)(answer)
# encoder rnn
encode_rnn = LSTM(n_hidden, return_sequences=True)(masked)
encode_rnn = LSTM(n_hidden, return_sequences=False)(encode_rnn)
encode_rnn = LSTM(n_hidden, return_sequences=True, dropout_U=0.2)(masked)
encode_rnn = LSTM(n_hidden, return_sequences=False, dropout_U=0.2)(encode_rnn)
encode_brnn = LSTM(n_hidden, return_sequences=True, go_backwards=True)(masked)
encode_brnn = LSTM(n_hidden, return_sequences=False, go_backwards=True)(encode_brnn)
encode_brnn = LSTM(n_hidden, return_sequences=True, go_backwards=True, dropout_U=0.2)(masked)
encode_brnn = LSTM(n_hidden, return_sequences=False, go_backwards=True, dropout_U=0.2)(encode_brnn)
# repeat it maxlen times
repeat_encoding = RepeatVector(question_maxlen)(encode_rnn)
repeat_encoding_rnn = RepeatVector(question_maxlen)(encode_rnn)
repeat_encoding_brnn = RepeatVector(question_maxlen)(encode_brnn)
# decoder rnn
decode_rnn = LSTM(n_hidden, return_sequences=True)(repeat_encoding)
decode_rnn = LSTM(n_hidden, return_sequences=True)(decode_rnn)
decode_rnn = LSTM(n_hidden, return_sequences=True, dropout_U=0.2, dropout_W=0.5)(repeat_encoding_rnn)
decode_rnn = LSTM(n_hidden, return_sequences=True, dropout_U=0.2)(decode_rnn)
decode_brnn = LSTM(n_hidden, return_sequences=True, go_backwards=True)(repeat_encoding)
decode_brnn = LSTM(n_hidden, return_sequences=True, go_backwards=True)(decode_brnn)
decode_brnn = LSTM(n_hidden, return_sequences=True, go_backwards=True, dropout_U=0.2, dropout_W=0.5)(repeat_encoding_brnn)
decode_brnn = LSTM(n_hidden, return_sequences=True, go_backwards=True, dropout_U=0.2)(decode_brnn)
merged_output = merge([decode_rnn, decode_brnn], mode='concat', concat_axis=-1)
# output
dense = TimeDistributed(Dense(vocab_len))(merged_output)
softmax = Activation('softmax')(dense)
regularized = ActivityRegularization(l2=1)(dense)
softmax = Activation('softmax')(regularized)
# compile the model
# compile the prediction model
model = Model([answer], [softmax])
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
if __name__ == '__main__':
question_maxlen, answer_maxlen = 10, 40
question_maxlen, answer_maxlen = 20, 60
qa = InsuranceQA()
batch_size = 50
@@ -93,30 +108,37 @@ if __name__ == '__main__':
print('Generating data...')
answers = qa.load('answers')
questions = qa.load('train')
def gen_questions(batch_size):
def gen_questions(batch_size, test=False):
if test:
questions = qa.load('test1')
else:
questions = qa.load('train')
while True:
i = 0
question_idx = np.zeros(shape=(batch_size, question_maxlen, len(qa.vocab)))
answer_idx = np.zeros(shape=(batch_size, answer_maxlen, len(qa.vocab)))
for s in questions:
a = s['answers'][0]
answer = qa.table.encode([qa.vocab[x] for x in answers[a]], answer_maxlen)
question = qa.table.encode([qa.vocab[x] for x in s['question']], question_maxlen)
answer_idx[i] = answer
question_idx[i] = question
i += 1
if i == batch_size:
yield ([answer_idx], [question_idx])
i = 0
if test:
ans = s['good']
else:
ans = s['answers']
for a in ans:
answer = qa.table.encode([qa.vocab[x] for x in answers[a]], answer_maxlen)
question = qa.table.encode([qa.vocab[x] for x in s['question']], question_maxlen)
question = np.amax(question, axis=0, keepdims=False)
answer_idx[i] = answer
question_idx[i] = question
i += 1
if i == batch_size:
yield ([answer_idx], [question_idx])
i = 0
gen = gen_questions(batch_size)
test_gen = gen_questions(n_test)
test_gen = gen_questions(n_test, test=True)
print('Generating model...')
model = get_model(question_maxlen=question_maxlen, answer_maxlen=answer_maxlen,
vocab_len=len(qa.vocab), n_hidden=128)
model = get_model(question_maxlen=question_maxlen, answer_maxlen=answer_maxlen, vocab_len=len(qa.vocab), n_hidden=128)
print('Training model...')
for iteration in range(1, 200):
@@ -128,7 +150,7 @@ if __name__ == '__main__':
x, y = next(test_gen)
y = y[0]
pred = model.predict(x, verbose=0)
for noise in [0, 0.1, 0.2]: # not sure what noise values would be good
for noise in [0.2, 0.5, 1.0, 1.2]: # not sure what noise values would be good
print(' Noise: {}'.format(noise))
for i in range(n_test):
print(' Expected: {}'.format(qa.table.decode(y[i])))
seq2seq/seq2seq.py
+30
View File
@@ -0,0 +1,30 @@
import keras.backend as K
from keras.engine import Input
from keras.layers import LSTM, TimeDistributed, Dense
class Seq2Seq:
def __init__(self, encode_seq_length, decode_seq_length, n_symbols, **params):
self.input = Input(shape=(encode_seq_length, n_symbols,))
self.encode_seq_length = encode_seq_length
self.decode_seq_length = decode_seq_length
self.n_symbols = n_symbols
self.params = params
encoder = self.build_encoder(self.input)
decoder = self.build_decoder(encoder)
def get_param(self, param, default):
if param in self.params:
return self.params[param]
print('Could not find param "{}" in params: Using default value {}'.format(param, default))
return default
def build_encoder(self, input):
lstm = LSTM(self.get_param('n_lstm_dims', 100), return_sequences=False)(input)
dense = Dense(self.n_symbols)(lstm)
return dense
def build_decoder(self, input):
pass