wassname/keras-language-modeling
1
0
Fork 0
mirror of https://github.com/wassname/keras-language-modeling.git synced 2026-09-10 12:15:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

generated recurrent transformations

Browse Source
This commit is contained in:
codekansas
2016-07-25 12:39:27 -07:00
parent b70eb2f7b6
commit 40d7cf84de
2 changed files with 52 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
insurance_qa_eval.py
+15 -11
View File
@@ -10,6 +10,8 @@ import pickle
from scipy.stats import rankdata
from keras_models import EmbeddingModel
random.seed(42)
@@ -23,7 +25,7 @@ class Evaluator:
self.path = data_path
self.conf = dict() if conf is None else conf
self.params = conf.get('training_params', dict())
self.answers = self.load('answers')
self.answers = self.load('generated') # self.load('answers')
self._vocab = None
self._reverse_vocab = None
self._eval_sets = None
@@ -132,6 +134,8 @@ class Evaluator:
if save_every is not None and i % save_every == 0:
self.save_epoch(model, i)
return val_loss
##### Evaluation #####
def prog_bar(self, so_far, total, n_bars=20):
@@ -226,13 +230,13 @@ if __name__ == '__main__':
'question_len': 20,
'answer_len': 60,
'n_words': 22353, # len(vocabulary) + 1
'margin': 0.05,
'margin': 0.009,
'training_params': {
'save_every': 1,
# 'eval_every': 1,
'batch_size': 128,
'nb_epoch': 1000,
'nb_epoch': 100,
'validation_split': 0.2,
'optimizer': 'adam',
# 'optimizer': Adam(clip_norm=0.1),
@@ -245,7 +249,7 @@ if __name__ == '__main__':
},
'model_params': {
'n_embed_dims': 100,
'n_embed_dims': 1000,
'n_hidden': 200,
# convolution
@@ -255,11 +259,11 @@ if __name__ == '__main__':
# recurrent
'n_lstm_dims': 141, # * 2
'initial_embed_weights': np.load('word2vec_100_dim.embeddings'),
# 'initial_embed_weights': np.load('word2vec_100_dim.embeddings'),
},
'similarity_params': {
'mode': 'gesd',
'mode': 'cosine',
'gamma': 1,
'c': 1,
'd': 2,
@@ -269,8 +273,7 @@ if __name__ == '__main__':
evaluator = Evaluator(conf)
##### Define model ######
from seq2seq.answer_to_question import EmbeddingRNNModel
model = EmbeddingRNNModel(conf)
model = EmbeddingModel(conf)
optimizer = conf.get('training_params', dict()).get('optimizer', 'adam')
model.compile(optimizer=optimizer)
@@ -284,8 +287,9 @@ if __name__ == '__main__':
# train the model
# evaluator.load_epoch(model, 54)
evaluator.train(model)
best_loss = evaluator.train(model)
# evaluate mrr for a particular epoch
# evaluator.load_epoch(model, 54)
# evaluator.get_mrr(model, evaluate_all=True)
evaluator.load_epoch(model, best_loss['epoch'])
# evaluator.load_epoch(model, 15)
evaluator.get_mrr(model, evaluate_all=True)
seq2seq/answer_to_question.py
+37 -25
View File
@@ -11,7 +11,7 @@ import random
import numpy as np
from keras.engine import Input
from keras.layers import RepeatVector, TimeDistributed, Dense, Activation, merge, GRU, Embedding, regularizers, Lambda, \
Dropout
constraints
from keras.models import Model
import keras.backend as K
@@ -34,6 +34,9 @@ class InsuranceQA:
def load(self, name):
return pickle.load(open(os.path.join(data_path, name), 'rb'))
def save(self, obj, name):
pickle.dump(obj, open(os.path.join(data_path, name), 'wb'))
class VocabularyTable:
def __init__(self, words):
self.words = sorted(set(words))
@@ -63,6 +66,8 @@ class InsuranceQA:
def get_model(question_maxlen, answer_maxlen, vocab_len, n_hidden, load_save=False):
answer = Input(shape=(answer_maxlen,), dtype='int32')
embedded = Embedding(input_dim=vocab_len, output_dim=n_hidden, mask_zero=True)(answer)
# answer = Input(shape=(answer_maxlen, vocab_len))
# embedded = Masking(mask_value=0.)(answer)
# encoder rnn
encode_rnn = GRU(n_hidden, return_sequences=True, dropout_U=0.2)(embedded)
@@ -116,24 +121,22 @@ class EmbeddingRNNModel(LanguageModel):
answer_argmax = argmax(answer_inverted)
# add embedding layers
# weights = self.model_params.get('initial_embed_weights', None)
# weights = weights if weights is None else [weights]
weights = self.model_params.get('initial_embed_weights', None)
weights = weights if weights is None else [weights]
embedding = Embedding(input_dim=self.config['n_words'],
output_dim=self.model_params.get('n_embed_dims', 100),
# weights=weights,
# W_regularizer=regularizers.activity_l1(1e-4),
W_constraint=constraints.nonneg(),
dropout=0.5,
weights=weights,
mask_zero=True)
question_embedding = embedding(question)
answer_embedding = embedding(answer_argmax)
# dropout
dropout = Dropout(0.5)
question_dropout = dropout(question_embedding)
answer_dropout = dropout(answer_embedding)
# maxpooling
maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]))
question_maxpool = maxpool(question_dropout)
answer_maxpool = maxpool(answer_dropout)
question_maxpool = maxpool(question_embedding)
answer_maxpool = maxpool(answer_embedding)
# activation
activation = Activation('tanh')
@@ -190,18 +193,27 @@ if __name__ == '__main__':
model = get_model(question_maxlen=question_maxlen, answer_maxlen=answer_maxlen, vocab_len=len(qa.vocab) + 1,
n_hidden=256, load_save=True)
print('Training model...')
for iteration in range(1, nb_iteration + 1):
print('\n' + '-' * 50 + '\nIteration %d' % iteration)
model.fit_generator(gen, samples_per_epoch=100 * batch_size, nb_epoch=nb_epoch)
model.save_weights(model_save, overwrite=True)
# print('Training model...')
# for iteration in range(1, nb_iteration + 1):
# print('\n' + '-' * 50 + '\nIteration %d' % iteration)
# model.fit_generator(gen, samples_per_epoch=100 * batch_size, nb_epoch=nb_epoch)
# model.save_weights(model_save, overwrite=True)
#
# # test this iteration on some sample data
# x, y = next(test_gen)
# pred = model.predict(x, verbose=0)
# y = y[0]
# x = x[0]
# for i in range(n_test):
# print('Answer: {}'.format(qa.table.decode(x[i], calc_argmax=False)))
# print(' Expected: {}'.format(qa.table.decode(y[i])))
# print(' Predicted: {}'.format(qa.table.decode(pred[i])))
# test this iteration on some sample data
x, y = next(test_gen)
pred = model.predict(x, verbose=0)
y = y[0]
x = x[0]
for i in range(n_test):
print('Answer: {}'.format(qa.table.decode(x[i], calc_argmax=False)))
print(' Expected: {}'.format(qa.table.decode(y[i])))
print(' Predicted: {}'.format(qa.table.decode(pred[i])))
print('Saving data points...')
generated = dict()
for key, answer in answers.items():
print('\r%d / %d' % (key, len(answers)), end = '')
output = model.predict(qa.table.encode([qa.vocab[x] for x in answer], answer_maxlen, one_hot=False).reshape((1, answer_maxlen)))
argmax = np.argmax(output, axis=-1)[0]
generated[key] = answer
qa.save(generated, 'generated')