wassname/keras-language-modeling
1
0
Fork 0
mirror of https://github.com/wassname/keras-language-modeling.git synced 2026-09-10 12:15:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

not sure what i changed

Browse Source
This commit is contained in:
codekansas
2016-07-31 13:18:08 -07:00
parent cb640bfe69
commit 8631a1c698
3 changed files with 24 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
insurance_qa_eval.py
+9 -14
View File
@@ -26,7 +26,7 @@ class Evaluator:
self.path = data_path
self.conf = dict() if conf is None else conf
self.params = conf.get('training_params', dict())
self.answers = self.load('answers') # self.load('generated')
self.answers = self.load('answers') # self.load('generated')
self._vocab = None
self._reverse_vocab = None
self._eval_sets = None
@@ -88,7 +88,6 @@ class Evaluator:
print(strftime('%Y-%m-%d %H:%M:%S :: ', gmtime()), end='')
def train(self, model):
eval_every = self.params.get('eval_every', None)
save_every = self.params.get('save_every', None)
batch_size = self.params.get('batch_size', 128)
nb_epoch = self.params.get('nb_epoch', 10)
@@ -112,11 +111,6 @@ class Evaluator:
# sample from all answers to get bad answers
bad_answers = self.pada(random.sample(self.answers.values(), len(good_answers)))
# shuffle questions
zipped = zip(questions, good_answers)
random.shuffle(zipped)
questions[:], good_answers[:] = zip(*zipped)
print('Epoch %d :: ' % i, end='')
self.print_time()
hist = model.fit([questions, good_answers, bad_answers], nb_epoch=1, batch_size=batch_size,
@@ -177,7 +171,7 @@ class Evaluator:
max_r = np.argmax(r)
max_n = np.argmax(r[:n_good])
# print(' '.join(self.revert(d['question'])))
# print(' '.join(self.revegrt(d['question'])))
# print(' '.join(self.revert(self.answers[indices[max_r]])))
# print(' '.join(self.revert(self.answers[indices[max_n]])))
@@ -222,10 +216,10 @@ if __name__ == '__main__':
import numpy as np
conf = {
'question_len': 200,
'answer_len': 200,
'question_len': 50,
'answer_len': 100,
'n_words': 22353, # len(vocabulary) + 1
'margin': 0.2,
'margin': 0.009,
'training_params': {
'save_every': 1,
@@ -247,6 +241,7 @@ if __name__ == '__main__':
'n_lstm_dims': 141, # * 2
'initial_embed_weights': np.load('models/word2vec_100_dim.h5'),
'similarity_dropout': 0.5,
},
'similarity_params': {
@@ -260,7 +255,7 @@ if __name__ == '__main__':
evaluator = Evaluator(conf)
##### Define model ######
model = AttentionModel(conf)
model = EmbeddingModel(conf)
optimizer = conf.get('training_params', dict()).get('optimizer', 'adam')
model.compile(optimizer=optimizer)
@@ -271,12 +266,12 @@ if __name__ == '__main__':
# np.save(open('models/embedding_1000_dim.h5', 'wb'), weights)
# train the model
# evaluator.load_epoch(model, 42)
# evaluator.load_epoch(model, 6)
best_loss = evaluator.train(model)
# evaluate mrr for a particular epoch
evaluator.load_epoch(model, best_loss['epoch'])
# evaluator.load_epoch(model, 31)
# evaluator.load_epoch(model, 68)
evaluator.get_mrr(model, evaluate_all=True)
# for epoch in range(1, 100):
# print('Epoch %d' % epoch)
keras_models.py
+15 -41
View File
@@ -117,7 +117,9 @@ class LanguageModel:
good_output = qa_model([self.question, self.answer_good])
bad_output = qa_model([self.question, self.answer_bad])
loss = merge([good_output, bad_output],
dropout = Dropout(self.model_params.get('similarity_dropout', 0.5))
loss = merge([dropout(good_output), dropout(bad_output)],
mode=lambda x: K.relu(self.config['margin'] - x[0] + x[1]),
output_shape=lambda x: x[0])
@@ -130,6 +132,9 @@ class LanguageModel:
def fit(self, x, **kwargs):
assert self.training_model is not None, 'Must compile the model before fitting data'
y = np.zeros(shape=x[0].shape[:1])
if 'validation_data' in kwargs:
data = kwargs['validation_data']
kwargs['validation_data'] = [data, np.zeros(shape=data[0].shape[:1])]
return self.training_model.fit(x, y, **kwargs)
def predict(self, x, **kwargs):
@@ -160,22 +165,12 @@ class EmbeddingModel(LanguageModel):
question_embedding = embedding(question)
answer_embedding = embedding(answer)
# dropout
dropout = Dropout(0.5)
question_dropout = dropout(question_embedding)
answer_dropout = dropout(answer_embedding)
# maxpooling
maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]))
question_maxpool = maxpool(question_dropout)
answer_maxpool = maxpool(answer_dropout)
question_pool = maxpool(question_embedding)
answer_pool = maxpool(answer_embedding)
# activation
activation = Activation('linear')
question_output = activation(question_maxpool)
answer_output = activation(answer_maxpool)
return question_output, answer_output
return question_pool, answer_pool
class ConvolutionModel(LanguageModel):
@@ -200,18 +195,13 @@ class ConvolutionModel(LanguageModel):
# embedding.params = []
# embedding.updates = []
# dropout
dropout = Dropout(0.5)
question_dropout = dropout(question_embedding)
answer_dropout = dropout(answer_embedding)
# dense
dense = TimeDistributed(Dense(self.model_params.get('n_hidden', 200),
# activity_regularizer=regularizers.activity_l1(1e-4),
# W_regularizer=regularizers.l1(1e-4),
activation='tanh'))
question_dense = dropout(dense(question_dropout))
answer_dense = dropout(dense(answer_dropout))
question_dense = dense(question_embedding)
answer_dense = dense(answer_embedding)
# cnn
cnns = [Convolution1D(filter_length=filter_length,
@@ -223,23 +213,13 @@ class ConvolutionModel(LanguageModel):
question_cnn = merge([cnn(question_dense) for cnn in cnns], mode='concat')
answer_cnn = merge([cnn(answer_dense) for cnn in cnns], mode='concat')
# dropout
question_dropout = dropout(question_cnn)
answer_dropout = dropout(answer_cnn)
# maxpooling
maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]))
avepool = Lambda(lambda x: K.mean(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]))
question_pool = maxpool(question_dropout)
answer_pool = maxpool(answer_dropout)
question_pool = maxpool(question_cnn)
answer_pool = maxpool(answer_cnn)
# activation
larger_dropout = Dropout(0.5)
activation = Activation('linear')
question_output = larger_dropout(activation(question_pool))
answer_output = larger_dropout(activation(answer_pool))
return question_output, answer_output
return question_pool, answer_pool
class AttentionModel(LanguageModel):
@@ -283,10 +263,4 @@ class AttentionModel(LanguageModel):
answer_b_rnn = b_rnn(answer_embedding)
answer_pool = merge([maxpool(answer_f_rnn), maxpool(answer_b_rnn)], mode='concat', concat_axis=-1)
# activation
dropout = Dropout(0.5)
activation = Activation('linear')
question_output = activation(dropout(question_pool))
answer_output = activation(dropout(answer_pool))
return question_output, answer_output
return question_pool, answer_pool
seq2seq/answer_to_question.py
-1
View File
@@ -127,7 +127,6 @@ class EmbeddingRNNModel(LanguageModel):
output_dim=self.model_params.get('n_embed_dims', 100),
# W_regularizer=regularizers.activity_l1(1e-4),
W_constraint=constraints.nonneg(),
dropout=0.5,
weights=weights,
mask_zero=True)
question_embedding = embedding(question)