diff --git a/insurance_qa_eval.py b/insurance_qa_eval.py index a915215..0e22b92 100644 --- a/insurance_qa_eval.py +++ b/insurance_qa_eval.py @@ -90,8 +90,9 @@ class Evaluator: for i in range(nb_epoch): # bad_answers = np.roll(good_answers, random.randint(10, len(questions) - 10)) - bad_answers = good_answers.copy() - random.shuffle(bad_answers) + # bad_answers = good_answers.copy() + # random.shuffle(bad_answers) + bad_answers = self.pada(random.sample(self.answers.values(), len(good_answers))) print('Epoch %d :: ' % i, end='') self.print_time() @@ -110,7 +111,7 @@ class Evaluator: self._eval_sets = dict([(s, self.load(s)) for s in ['dev', 'test1', 'test2']]) return self._eval_sets - def get_mrr(self, model, n_eval=20): + def get_mrr(self, model): top1s = list() mrrs = list() @@ -119,7 +120,9 @@ class Evaluator: print('----- %s -----' % name) random.shuffle(data) - data = data[:n_eval] + + if 'n_eval' in self.params: + data = data[:self.params['n_eval']] c_1, c_2 = 0, 0 @@ -152,20 +155,22 @@ class Evaluator: if __name__ == '__main__': conf = { - 'question_len': 10, - 'answer_len': 40, + 'question_len': 100, + 'answer_len': 100, 'n_words': 22353, # len(vocabulary) + 1 - 'margin': 0.2, + 'margin': 0.009, 'training_params': { 'eval_every': 10, 'save_every': None, 'batch_size': 128, 'nb_epoch': 100, + # 'n_eval': 20, }, 'model_params': { 'n_embed_dims': 1000, + 'n_hidden': 200, # convolution 'nb_filters': 1000, @@ -185,7 +190,7 @@ if __name__ == '__main__': evaluator = Evaluator(data_path, conf) ##### Define model ###### - model = ConvolutionModel(conf) + model = EmbeddingModel(conf) model.compile(optimizer='adam') evaluator.train(model) diff --git a/keras_models.py b/keras_models.py index 4e5e4b5..e68304e 100644 --- a/keras_models.py +++ b/keras_models.py @@ -3,7 +3,8 @@ from __future__ import print_function from abc import abstractmethod from keras.engine import Input -from keras.layers import merge, Embedding, Dropout, Convolution1D, Lambda, Activation, LSTM +from keras.layers import merge, Embedding, Dropout, Convolution1D, Lambda, Activation, LSTM, Dense, TimeDistributed, \ + ActivityRegularization from keras import backend as K from keras.models import Model @@ -37,13 +38,36 @@ class LanguageModel: return def get_similarity(self): + ''' Specify similarity in configuration under 'similarity_params' -> 'mode' + If a parameter is needed for the model, specify it in 'similarity_params' + + Example configuration: + + config = { + ... other parameters ... + 'similarity_params': { + 'mode': 'gesd', + 'gamma': 1, + 'c': 1, + } + } + + cosine: dot(a, b) / sqrt(dot(a, a) * dot(b, b)) + polynomial: (gamma * dot(a, b) + c) ^ d + sigmoid: tanh(gamma * dot(a, b) + c) + rbf: exp(-gamma * l2_norm(a-b) ^ 2) + euclidean: 1 / (1 + l2_norm(a - b)) + exponential: exp(-gamma * l2_norm(a - b)) + gesd: euclidean * sigmoid + aesd: (euclidean + sigmoid) / 2 + ''' + params = self.similarity_params similarity = params['mode'] axis = lambda a: len(a._keras_shape) - 1 - dot = lambda a, b: K.batch_dot(a, b, axes=axis(a)) - sum = lambda a, ax: K.sum(a, axis=ax, keepdims=True) + l2_norm = lambda a, b: K.sqrt(K.sum((a - b) ** 2, axis=axis(a), keepdims=True)) if similarity == 'cosine': return lambda x: dot(x[0], x[1]) / K.sqrt(dot(x[0], x[0]) * dot(x[1], x[1])) @@ -52,18 +76,18 @@ class LanguageModel: elif similarity == 'sigmoid': return lambda x: K.tanh(params['gamma'] * dot(x[0], x[1]) + params['c']) elif similarity == 'rbf': - return lambda x: K.exp(-1 * sum(x[0] - x[1], axis(x[0])) ** 2) + return lambda x: K.exp(-1 * params['gamma'] * l2_norm(x[0], x[1]) ** 2) elif similarity == 'euclidean': - return lambda x: 1 / (1 + sum(x[0] - x[1], axis(x[0]))) + return lambda x: 1 / (1 + l2_norm(x[0], x[1])) elif similarity == 'exponential': - return lambda x: K.exp(-1 * sum(K.abs(x[0] - x[1]), axis(x[0]))) + return lambda x: K.exp(-1 * params['gamma'] * l2_norm(x[0], x[1])) elif similarity == 'gesd': - euclidean = lambda x: 1 / (1 + sum(x[0] - x[1], axis(x[0]))) + euclidean = lambda x: 1 / (1 + l2_norm(x[0], x[1])) sigmoid = lambda x: 1 / (1 + K.exp(-1 * params['gamma'] * (dot(x[0], x[1]) + params['c']))) return lambda x: euclidean(x) * sigmoid(x) elif similarity == 'aesd': - euclidean = lambda x: 1 / (1 + sum(x[0] - x[1], axis(x[0]))) - sigmoid = lambda x: 1 / (1 + K.exp(-1 * params['gamma'] * (dot(x[0], x[1]) + params['c']))) + euclidean = lambda x: 0.5 / (1 + l2_norm(x[0], x[1])) + sigmoid = lambda x: 0.5 / (1 + K.exp(-1 * params['gamma'] * (dot(x[0], x[1]) + params['c']))) return lambda x: euclidean(x) + sigmoid(x) else: raise Exception('Invalid similarity: {}'.format(similarity)) @@ -125,6 +149,31 @@ class LanguageModel: self.training_model.load_weights(file_name, **kwargs) +class EmbeddingModel(LanguageModel): + def build(self): + input, _ = self._get_inputs() + + # add embedding layers + embedding = Embedding(self.config['n_words'], self.model_params.get('n_embed_dims', 141)) + input_embedding = embedding(input) + + # dropout + dropout = Dropout(0.5) + input_dropout = dropout(input_embedding) + + # maxpooling + maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2])) + input_pool = maxpool(input_dropout) + + # activation + activation = Activation('tanh') + output = activation(input_pool) + + model = Model(input=[input], output=[output]) + + return model, model + + class ConvolutionModel(LanguageModel): def build(self): input, _ = self._get_inputs() @@ -137,6 +186,16 @@ class ConvolutionModel(LanguageModel): dropout = Dropout(0.5) input_dropout = dropout(input_embedding) + # hidden layer + dense = TimeDistributed(Dense(self.model_params.get('n_hidden', 200), activation='tanh')) + input_dense = dense(input_dropout) + + # regularizer + input_dense = ActivityRegularization(l2=0.0001)(input_dense) + + # dropout + input_dropout = dropout(input_dense) + # cnn cnns = [Convolution1D(filter_length=filter_length, nb_filter=self.model_params.get('nb_filters', 1000),