wassname/keras-language-modeling
1
0
Fork 0
mirror of https://github.com/wassname/keras-language-modeling.git synced 2026-09-10 12:15:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactored some stuff

Browse Source
This commit is contained in:
codekansas
2016-07-26 10:21:06 -07:00
parent e803431796
commit cb640bfe69
3 changed files with 43 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
attention_lstm.py
+3 -5
View File
@@ -6,11 +6,9 @@ from keras.layers import LSTM, activations
class AttentionLSTM(LSTM):
def __init__(self, output_dim, attention_vec, attn_activation='tanh',
attn_inner_activation='tanh', single_attention_param=False,
n_attention_dim=None, **kwargs):
single_attention_param=False, n_attention_dim=None, **kwargs):
self.attention_vec = attention_vec
self.attn_activation = activations.get(attn_activation)
self.attn_inner_activation = activations.get(attn_inner_activation)
self.single_attention_param = single_attention_param
self.n_attention_dim = output_dim if n_attention_dim is None else n_attention_dim
@@ -51,10 +49,10 @@ class AttentionLSTM(LSTM):
h, [h, c] = super(AttentionLSTM, self).step(x, states)
attention = states[4]
m = self.attn_inner_activation(K.dot(h, self.U_a) * attention + self.b_a)
m = self.attn_activation(K.dot(h, self.U_a) * attention + self.b_a)
# Intuitively it makes more sense to use a sigmoid (was getting some NaN problems
# which I think might have been caused by the exponential function -> gradients blow up)
s = self.attn_activation(K.dot(m, self.U_s) + self.b_s)
s = K.sigmoid(K.dot(m, self.U_s) + self.b_s)
if self.single_attention_param:
h = h * K.repeat_elements(s, self.output_dim, axis=1)
insurance_qa_eval.py
+14 -27
View File
@@ -8,9 +8,10 @@ from time import strftime, gmtime
import pickle
from keras.optimizers import Adam
from scipy.stats import rankdata
from keras_models import EmbeddingModel, AttentionModel
from keras_models import EmbeddingModel, AttentionModel, ConvolutionModel
random.seed(42)
@@ -104,20 +105,17 @@ class Evaluator:
questions = self.padq(questions)
good_answers = self.pada(good_answers)
# bad_answers = self.pada(random.sample(self.answers.values(), len(good_answers)))
val_loss = {'loss': 1., 'epoch': 0}
for i in range(1, nb_epoch):
# bad_answers = np.roll(good_answers, random.randint(10, len(questions) - 10))
# bad_answers = good_answers.copy()
# random.shuffle(bad_answers)
# sample from all answers to get bad answers
bad_answers = self.pada(random.sample(self.answers.values(), len(good_answers)))
# shuffle questions
# zipped = zip(questions, good_answers)
# random.shuffle(zipped)
# questions[:], good_answers[:] = zip(*zipped)
zipped = zip(questions, good_answers)
random.shuffle(zipped)
questions[:], good_answers[:] = zip(*zipped)
print('Epoch %d :: ' % i, end='')
self.print_time()
@@ -128,9 +126,6 @@ class Evaluator:
val_loss = {'loss': hist.history['val_loss'][0], 'epoch': i}
print('Best: Loss = {}, Epoch = {}'.format(val_loss['loss'], val_loss['epoch']))
if eval_every is not None and i % eval_every == 0:
self.get_mrr(model)
if save_every is not None and i % save_every == 0:
self.save_epoch(model, i)
@@ -227,25 +222,17 @@ if __name__ == '__main__':
import numpy as np
conf = {
'question_len': 20,
'answer_len': 60,
'question_len': 200,
'answer_len': 200,
'n_words': 22353, # len(vocabulary) + 1
'margin': 0.2,
'training_params': {
'save_every': 1,
# 'eval_every': 1,
'batch_size': 20,
'nb_epoch': 1000,
'nb_epoch': 100,
'validation_split': 0.2,
'optimizer': 'adam',
# 'optimizer': Adam(clip_norm=0.1),
# 'n_eval': 100,
'evaluate_all_threshold': {
'mode': 'all',
'top1': 0.4,
},
'optimizer': Adam(clipnorm=1e-2),
},
'model_params': {
@@ -254,7 +241,7 @@ if __name__ == '__main__':
# convolution
'nb_filters': 1000, # * 4
'conv_activation': 'relu',
'conv_activation': 'tanh',
# recurrent
'n_lstm_dims': 141, # * 2
@@ -263,7 +250,7 @@ if __name__ == '__main__':
},
'similarity_params': {
'mode': 'cosine',
'mode': 'gesd',
'gamma': 1,
'c': 1,
'd': 2,
@@ -284,12 +271,12 @@ if __name__ == '__main__':
# np.save(open('models/embedding_1000_dim.h5', 'wb'), weights)
# train the model
# evaluator.load_epoch(model, 54)
# evaluator.load_epoch(model, 42)
best_loss = evaluator.train(model)
# evaluate mrr for a particular epoch
evaluator.load_epoch(model, best_loss['epoch'])
# evaluator.load_epoch(model, 116)
# evaluator.load_epoch(model, 31)
evaluator.get_mrr(model, evaluate_all=True)
# for epoch in range(1, 100):
# print('Epoch %d' % epoch)
keras_models.py
+26 -34
View File
@@ -118,7 +118,7 @@ class LanguageModel:
bad_output = qa_model([self.question, self.answer_bad])
loss = merge([good_output, bad_output],
mode=lambda x: K.maximum(1e-6, self.config['margin'] - x[0] + x[1]),
mode=lambda x: K.relu(self.config['margin'] - x[0] + x[1]),
output_shape=lambda x: x[0])
self.training_model = Model(input=[self.question, self.answer_good, self.answer_bad], output=loss)
@@ -201,24 +201,24 @@ class ConvolutionModel(LanguageModel):
# embedding.updates = []
# dropout
dropout = Dropout(0.25)
dropout = Dropout(0.5)
question_dropout = dropout(question_embedding)
answer_dropout = dropout(answer_embedding)
# dense
dense = TimeDistributed(Dense(self.model_params.get('n_hidden', 200), activation='tanh',
activity_regularizer=regularizers.activity_l1(1e-4),
W_regularizer=regularizers.l1(1e-4),
dropout=0.5))
question_dense = dense(question_dropout)
answer_dense = dense(answer_dropout)
dense = TimeDistributed(Dense(self.model_params.get('n_hidden', 200),
# activity_regularizer=regularizers.activity_l1(1e-4),
# W_regularizer=regularizers.l1(1e-4),
activation='tanh'))
question_dense = dropout(dense(question_dropout))
answer_dense = dropout(dense(answer_dropout))
# cnn
cnns = [Convolution1D(filter_length=filter_length,
nb_filter=self.model_params.get('nb_filters', 1000),
activation=self.model_params.get('conv_activation', 'relu'),
W_regularizer=regularizers.l1(1e-4),
activity_regularizer=regularizers.activity_l1(1e-4),
# W_regularizer=regularizers.l1(1e-4),
# activity_regularizer=regularizers.activity_l1(1e-4),
border_mode='same') for filter_length in [2, 3, 5, 7]]
question_cnn = merge([cnn(question_dense) for cnn in cnns], mode='concat')
answer_cnn = merge([cnn(answer_dense) for cnn in cnns], mode='concat')
@@ -229,14 +229,15 @@ class ConvolutionModel(LanguageModel):
# maxpooling
maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]))
avepool = Lambda(lambda x: K.mean(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]))
question_pool = maxpool(question_dropout)
answer_pool = maxpool(answer_dropout)
# activation
dropout = Dropout(0.4)
larger_dropout = Dropout(0.5)
activation = Activation('linear')
question_output = activation(dropout(question_pool))
answer_output = activation(dropout(answer_pool))
question_output = larger_dropout(activation(question_pool))
answer_output = larger_dropout(activation(answer_pool))
return question_output, answer_output
@@ -260,39 +261,30 @@ class AttentionModel(LanguageModel):
# embedding.params = []
# embedding.updates = []
# dropout
dropout = Dropout(0.25)
question_dropout = dropout(question_embedding)
answer_dropout = dropout(answer_embedding)
# question rnn part
f_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True, dropout_U=0.2,
U_regularizer=regularizers.l2(1e-4), consume_less='mem')
consume_less='mem')
b_rnn = LSTM(self.model_params.get('n_lstm_dims', 141), return_sequences=True, dropout_U=0.2,
U_regularizer=regularizers.l2(1e-4), consume_less='mem', go_backwards=True)
question_f_rnn = f_rnn(question_dropout)
question_b_rnn = b_rnn(question_dropout)
question_f_dropout = dropout(question_f_rnn)
question_b_dropout = dropout(question_b_rnn)
consume_less='mem', go_backwards=True)
question_f_rnn = f_rnn(question_embedding)
question_b_rnn = b_rnn(question_embedding)
# maxpooling
maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]))
avepool = Lambda(lambda x: K.mean(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]))
question_pool = merge([avepool(question_f_dropout), avepool(question_b_dropout)], mode='concat', concat_axis=-1)
question_pool = merge([maxpool(question_f_rnn), maxpool(question_b_rnn)], mode='concat', concat_axis=-1)
# answer rnn part
f_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, dropout_U=0.2,
U_regularizer=regularizers.l1(1e-4), return_sequences=True, consume_less='mem',
single_attention_param=True)
b_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, dropout_U=0.2,
U_regularizer=regularizers.l1(1e-4), return_sequences=True, consume_less='mem',
go_backwards=True, single_attention_param=True)
answer_f_rnn = f_rnn(answer_dropout)
answer_b_rnn = b_rnn(answer_dropout)
f_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, return_sequences=True,
consume_less='mem', single_attention_param=True)
b_rnn = AttentionLSTM(self.model_params.get('n_lstm_dims', 141), question_pool, return_sequences=True,
consume_less='mem', go_backwards=True, single_attention_param=True)
answer_f_rnn = f_rnn(answer_embedding)
answer_b_rnn = b_rnn(answer_embedding)
answer_pool = merge([maxpool(answer_f_rnn), maxpool(answer_b_rnn)], mode='concat', concat_axis=-1)
# activation
dropout = Dropout(0.4)
dropout = Dropout(0.5)
activation = Activation('linear')
question_output = activation(dropout(question_pool))
answer_output = activation(dropout(answer_pool))