wassname/keras-language-modeling
1
0
Fork 0
mirror of https://github.com/wassname/keras-language-modeling.git synced 2026-09-12 12:33:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed similarities

Browse Source
This commit is contained in:
codekansas
2016-04-30 01:04:53 -04:00
parent 3a9e66f706
commit ef2c257268
2 changed files with 81 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
insurance_qa_eval.py
+13 -8
View File
@@ -90,8 +90,9 @@ class Evaluator:
for i in range(nb_epoch):
# bad_answers = np.roll(good_answers, random.randint(10, len(questions) - 10))
bad_answers = good_answers.copy()
random.shuffle(bad_answers)
# bad_answers = good_answers.copy()
# random.shuffle(bad_answers)
bad_answers = self.pada(random.sample(self.answers.values(), len(good_answers)))
print('Epoch %d :: ' % i, end='')
self.print_time()
@@ -110,7 +111,7 @@ class Evaluator:
self._eval_sets = dict([(s, self.load(s)) for s in ['dev', 'test1', 'test2']])
return self._eval_sets
def get_mrr(self, model, n_eval=20):
def get_mrr(self, model):
top1s = list()
mrrs = list()
@@ -119,7 +120,9 @@ class Evaluator:
print('----- %s -----' % name)
random.shuffle(data)
data = data[:n_eval]
if 'n_eval' in self.params:
data = data[:self.params['n_eval']]
c_1, c_2 = 0, 0
@@ -152,20 +155,22 @@ class Evaluator:
if __name__ == '__main__':
conf = {
'question_len': 10,
'answer_len': 40,
'question_len': 100,
'answer_len': 100,
'n_words': 22353, # len(vocabulary) + 1
'margin': 0.2,
'margin': 0.009,
'training_params': {
'eval_every': 10,
'save_every': None,
'batch_size': 128,
'nb_epoch': 100,
# 'n_eval': 20,
},
'model_params': {
'n_embed_dims': 1000,
'n_hidden': 200,
# convolution
'nb_filters': 1000,
@@ -185,7 +190,7 @@ if __name__ == '__main__':
evaluator = Evaluator(data_path, conf)
##### Define model ######
model = ConvolutionModel(conf)
model = EmbeddingModel(conf)
model.compile(optimizer='adam')
evaluator.train(model)
keras_models.py
+68 -9
View File
@@ -3,7 +3,8 @@ from __future__ import print_function
from abc import abstractmethod
from keras.engine import Input
from keras.layers import merge, Embedding, Dropout, Convolution1D, Lambda, Activation, LSTM
from keras.layers import merge, Embedding, Dropout, Convolution1D, Lambda, Activation, LSTM, Dense, TimeDistributed, \
ActivityRegularization
from keras import backend as K
from keras.models import Model
@@ -37,13 +38,36 @@ class LanguageModel:
return
def get_similarity(self):
''' Specify similarity in configuration under 'similarity_params' -> 'mode'
If a parameter is needed for the model, specify it in 'similarity_params'
Example configuration:
config = {
... other parameters ...
'similarity_params': {
'mode': 'gesd',
'gamma': 1,
'c': 1,
}
}
cosine: dot(a, b) / sqrt(dot(a, a) * dot(b, b))
polynomial: (gamma * dot(a, b) + c) ^ d
sigmoid: tanh(gamma * dot(a, b) + c)
rbf: exp(-gamma * l2_norm(a-b) ^ 2)
euclidean: 1 / (1 + l2_norm(a - b))
exponential: exp(-gamma * l2_norm(a - b))
gesd: euclidean * sigmoid
aesd: (euclidean + sigmoid) / 2
'''
params = self.similarity_params
similarity = params['mode']
axis = lambda a: len(a._keras_shape) - 1
dot = lambda a, b: K.batch_dot(a, b, axes=axis(a))
sum = lambda a, ax: K.sum(a, axis=ax, keepdims=True)
l2_norm = lambda a, b: K.sqrt(K.sum((a - b) ** 2, axis=axis(a), keepdims=True))
if similarity == 'cosine':
return lambda x: dot(x[0], x[1]) / K.sqrt(dot(x[0], x[0]) * dot(x[1], x[1]))
@@ -52,18 +76,18 @@ class LanguageModel:
elif similarity == 'sigmoid':
return lambda x: K.tanh(params['gamma'] * dot(x[0], x[1]) + params['c'])
elif similarity == 'rbf':
return lambda x: K.exp(-1 * sum(x[0] - x[1], axis(x[0])) ** 2)
return lambda x: K.exp(-1 * params['gamma'] * l2_norm(x[0], x[1]) ** 2)
elif similarity == 'euclidean':
return lambda x: 1 / (1 + sum(x[0] - x[1], axis(x[0])))
return lambda x: 1 / (1 + l2_norm(x[0], x[1]))
elif similarity == 'exponential':
return lambda x: K.exp(-1 * sum(K.abs(x[0] - x[1]), axis(x[0])))
return lambda x: K.exp(-1 * params['gamma'] * l2_norm(x[0], x[1]))
elif similarity == 'gesd':
euclidean = lambda x: 1 / (1 + sum(x[0] - x[1], axis(x[0])))
euclidean = lambda x: 1 / (1 + l2_norm(x[0], x[1]))
sigmoid = lambda x: 1 / (1 + K.exp(-1 * params['gamma'] * (dot(x[0], x[1]) + params['c'])))
return lambda x: euclidean(x) * sigmoid(x)
elif similarity == 'aesd':
euclidean = lambda x: 1 / (1 + sum(x[0] - x[1], axis(x[0])))
sigmoid = lambda x: 1 / (1 + K.exp(-1 * params['gamma'] * (dot(x[0], x[1]) + params['c'])))
euclidean = lambda x: 0.5 / (1 + l2_norm(x[0], x[1]))
sigmoid = lambda x: 0.5 / (1 + K.exp(-1 * params['gamma'] * (dot(x[0], x[1]) + params['c'])))
return lambda x: euclidean(x) + sigmoid(x)
else:
raise Exception('Invalid similarity: {}'.format(similarity))
@@ -125,6 +149,31 @@ class LanguageModel:
self.training_model.load_weights(file_name, **kwargs)
class EmbeddingModel(LanguageModel):
def build(self):
input, _ = self._get_inputs()
# add embedding layers
embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 141))
input_embedding = embedding(input)
# dropout
dropout = Dropout(0.5)
input_dropout = dropout(input_embedding)
# maxpooling
maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]))
input_pool = maxpool(input_dropout)
# activation
activation = Activation('tanh')
output = activation(input_pool)
model = Model(input=[input], output=[output])
return model, model
class ConvolutionModel(LanguageModel):
def build(self):
input, _ = self._get_inputs()
@@ -137,6 +186,16 @@ class ConvolutionModel(LanguageModel):
dropout = Dropout(0.5)
input_dropout = dropout(input_embedding)
# hidden layer
dense = TimeDistributed(Dense(self.model_params.get('n_hidden', 200), activation='tanh'))
input_dense = dense(input_dropout)
# regularizer
input_dense = ActivityRegularization(l2=0.0001)(input_dense)
# dropout
input_dropout = dropout(input_dense)
# cnn
cnns = [Convolution1D(filter_length=filter_length,
nb_filter=self.model_params.get('nb_filters', 1000),