wassname/keras-language-modeling
1
0
Fork 0
mirror of https://github.com/wassname/keras-language-modeling.git synced 2026-09-10 12:15:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

ran some more trials

Browse Source
This commit is contained in:
codekansas
2016-05-02 20:30:28 -04:00
parent 9ff917d3ac
commit 8946dbccf4
3 changed files with 104 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
insurance_qa_eval.py
+58 -13
View File
@@ -107,7 +107,7 @@ class Evaluator:
# random.shuffle(bad_answers)
bad_answers = self.pada(random.sample(self.answers.values(), len(good_answers)))
print('Epoch %d :: ' % i, end='')
print('Epoch %d :: ' % (i+1), end='')
self.print_time()
model.fit([questions, good_answers, bad_answers], nb_epoch=1, batch_size=batch_size, validation_split=split)
@@ -119,32 +119,45 @@ class Evaluator:
##### Evaluation #####
def prog_bar(self, so_far, total, n_bars=20):
n_complete = int(so_far * n_bars / total)
if n_complete >= n_bars - 1:
print('\r[' + '=' * n_bars + ']', end='')
else:
s = '\r[' + '=' * (n_complete - 1) + '>' + '.' * (n_bars - n_complete) + ']'
print(s, end='')
def eval_sets(self):
if self._eval_sets is None:
self._eval_sets = dict([(s, self.load(s)) for s in ['dev', 'test1', 'test2']])
return self._eval_sets
def get_mrr(self, model):
def get_mrr(self, model, evaluate_all=False):
top1s = list()
mrrs = list()
for name, data in self.eval_sets().items():
self.print_time()
print('----- %s -----' % name)
if evaluate_all:
self.print_time()
print('----- %s -----' % name)
random.shuffle(data)
if 'n_eval' in self.params:
if not evaluate_all and 'n_eval' in self.params:
data = data[:self.params['n_eval']]
c_1, c_2 = 0, 0
for d in data:
c = 0
for i, d in enumerate(data):
if evaluate_all:
self.prog_bar(i, len(data))
answers = self.pada([self.answers[i] for i in d['good'] + d['bad']])
question = self.padq([d['question']] * len(d['good'] + d['bad']))
n_good = len(d['good'])
sims = model.predict([question, answers], batch_size=300).flatten()
sims = model.predict([question, answers], batch_size=500).flatten()
r = rankdata(sims, method='max')
max_r = np.argmax(r)
@@ -158,12 +171,32 @@ class Evaluator:
del data
print('Top-1 Precision: %f' % top1)
print('MRR: %f' % mrr)
if evaluate_all:
print('Top-1 Precision: %f' % top1)
print('MRR: %f' % mrr)
top1s.append(top1)
mrrs.append(mrr)
# rerun the evaluation if above some threshold
if not evaluate_all:
print('Top-1 Precision: {}'.format(top1s))
print('MRR: {}'.format(mrrs))
evaluate_all_threshold = self.params.get('evaluate_all_threshold', dict())
evaluate_mode = evaluate_all_threshold.get('mode', 'all')
mrr_theshold = evaluate_all_threshold.get('mrr', 1)
top1_threshold = evaluate_all_threshold.get('top1', 1)
if evaluate_mode == 'any':
evaluate_all = evaluate_all or any([x > top1_threshold for x in top1s])
evaluate_all = evaluate_all or any([x > mrr_theshold for x in mrrs])
else:
evaluate_all = evaluate_all or all([x > top1_threshold for x in top1s])
evaluate_all = evaluate_all or all([x > mrr_theshold for x in mrrs])
if evaluate_all:
return self.get_mrr(model, evaluate_all=True)
return top1s, mrrs
if __name__ == '__main__':
@@ -175,16 +208,21 @@ if __name__ == '__main__':
'training_params': {
'save_every': 1,
# 'eval_every': 20,
'eval_every': 1,
'batch_size': 128,
'nb_epoch': 1000,
'validation_split': 0.2,
'optimizer': 'adam',
# 'n_eval': 20,
'n_eval': 20,
'evaluate_all_threshold': {
'mode': 'any',
'top1': 0.55,
},
},
'model_params': {
'n_embed_dims': 1000,
'n_embed_dims': 100,
'n_hidden': 200,
# convolution
@@ -210,9 +248,16 @@ if __name__ == '__main__':
optimizer = conf.get('training_params', dict()).get('optimizer', 'adam')
model.compile(optimizer=optimizer)
# load pre-trained embedding layer
import numpy as np
weights = np.load('models/embedding_100_dim.h5')
language_model = model.prediction_model.layers[2]
language_model.layers[2].set_weights([weights])
# train the model
evaluator.load_epoch(model, 10)
evaluator.train(model)
# evaluate mrr for a particular epoch
# evaluator.load_epoch(model, -1)
# evaluator.load_epoch(model, 10)
# evaluator.get_mrr(model)
keras_models.py
+20 -6
View File
@@ -136,12 +136,12 @@ class LanguageModel:
return self.prediction_model.predict(x, **kwargs)
def save_weights(self, file_name, **kwargs):
assert self.training_model is not None, 'Must compile the model before saving weights'
self.training_model.save_weights(file_name, **kwargs)
assert self.prediction_model is not None, 'Must compile the model before saving weights'
self.prediction_model.save_weights(file_name, **kwargs)
def load_weights(self, file_name, **kwargs):
assert self.training_model is not None, 'Must compile the model loading weights'
self.training_model.load_weights(file_name, **kwargs)
assert self.prediction_model is not None, 'Must compile the model loading weights'
self.prediction_model.load_weights(file_name, **kwargs)
class EmbeddingModel(LanguageModel):
@@ -181,6 +181,8 @@ class ConvolutionModel(LanguageModel):
return merge([cnn(input) for cnn in cnns], mode='concat'), cnns
def build(self):
assert self.config['question_len'] == self.config['answer_len']
question, answer = self._get_inputs()
# add embedding layers
@@ -188,8 +190,12 @@ class ConvolutionModel(LanguageModel):
question_embedding = embedding(question)
answer_embedding = embedding(answer)
# turn off layer updating
embedding.params = []
embedding.updates = []
# dropout
dropout = Dropout(0.5)
dropout = Dropout(0.25)
question_dropout = dropout(question_embedding)
answer_dropout = dropout(answer_embedding)
@@ -198,6 +204,10 @@ class ConvolutionModel(LanguageModel):
question_dense = dense(question_dropout)
answer_dense = dense(answer_dropout)
# regularization
question_dense = ActivityRegularization(l2=0.0001)(question_dense)
answer_dense = ActivityRegularization(l2=0.0001)(answer_dense)
# dropout
question_dropout = dropout(question_dense)
answer_dropout = dropout(answer_dense)
@@ -206,10 +216,14 @@ class ConvolutionModel(LanguageModel):
cnns = [Convolution1D(filter_length=filter_length,
nb_filter=self.model_params.get('nb_filters', 1000),
activation=self.model_params.get('conv_activation', 'relu'),
border_mode='same') for filter_length in [2, 3, 4, 5]]
border_mode='same') for filter_length in [2, 3, 5, 7]]
question_cnn = merge([cnn(question_dropout) for cnn in cnns], mode='concat')
answer_cnn = merge([cnn(answer_dropout) for cnn in cnns], mode='concat')
# regularization
question_cnn = ActivityRegularization(l2=0.0001)(question_cnn)
answer_cnn = ActivityRegularization(l2=0.0001)(answer_cnn)
# dropout
question_dropout = dropout(question_cnn)
answer_dropout = dropout(answer_cnn)
results.notes
+26 -32
View File
@@ -4,32 +4,6 @@ Single-layer bi-LSTM with max pooling, 40 words per sentence, loss margin of 0.2
- Using the pure embedding layer worked better than using the Word2Vec model (gave MRR ~0.09)
CNN + ALSTM model seems to have good loss
- MRR ~0.33 (I think)
CNN + ALSTM
- Example error:
- Question: be home insurance negotiable
- Desired answer: I not sure exactly what you mean but if you mean go without then no a mortgage company will not allow
that and if you own outright it will be a foolish choice go without if you mean price negotiation (ranked 177)
- Highest-rank answer: no homeowner be not mandatory in Pennsylvania however if you be apply for any type of homeowner
loan most bank will require you purchase insurance process the loan the bank will also ask be list as the lien holder if
- Example error:
- Question: what be Suze Orman advice on long term care insurance
- Desired answer: Suze Orman recommend long-term care insurance if it fit and 1 can medically qualify for it she be
currently take care of her 90 year old mother who refuse to letSuze buy long-term care insurance on her when she can
(rank ~460)
- Highest-rank answer: a good age buy long-term care insurance when you be in a good financial position and have some
extra disposable income also if your health be good you will save thousand dollar potentially in premium need specific
number how about
- Example error:
- Question: can you get Life Insurance on someone else
- Desired answer: you can get life insurance on someone else if you have an insurable interest with them an example of
insurable interest be this : if a nonrelative owe you money you can take out a life insurance policy on them (rank 498)
- Highest-rank answer: I assume when you ask Doe someone have a life insurance policy on me , you be ask about someone
other than your parent as an adult no one can take a policy out on you without you give your
- In general, the model predicts the topics well, but doesn't necessarily match it well with the question. This might be due to the masking issue.
- MRR: 0.30957
@@ -42,18 +16,14 @@ Pure CNN Model:
- After 120 epochs, InsuranceQA MRR = 0.530 and Top-1 Precision = 0.401
- Decreased slightly with further training due to overfitting
- Mixing together different filter lengths improved it to MRR = 0.56 and Top-1 Precision = 0.44
- Example error:
- Question: how much do Disability Income insurance cost
- Desired answer: disability income insurance can cost between $50 and $5,000 per month yes that be a ridiculously big range but there be so many variable that determine the rate , I shall cover all base a full-time broker / agent can quickly ask you a few question and determine the good plan for the most affordable price
- Highest-rank answer: along with what Peggy and Steve mention make sure whoever you choose as your agent present you with multiple quote for disability insurance too often consumer think the cheap policy be good but it have likely that the cheap policy offer poor coverage your agent assume he or she do not represent just 1 company shall be able provide you different quote that reflect different type of coverage ( e.g. different benefit period , elimination period , different company , etc.
- Rank of best answer: 347.0
- Validation loss around 6e-4 (lower than maxpooling) underperformed compared to maxpooling
Embedding + MaxPooling:
- I can't believe this model performed so well. It blew the other ones out of the water, and trains ridiculously quickly.
- Test 1: Top-1 Precision = 0.4922, MRR = 0.6239
- Test 2: Top-1 Precision = 0.4817, MRR = 0.6110
- Dev: Top-1 Precision = 0.4950, MRR = 0.6244
- With 100 dimensions, Top-1 Precision = 0.281, MRR = 0.417 on test 1
- Adding more embedding dimensions (beyond 1000) didn't lead to an improvement
- Converted after about 20 epochs
- Validation loss was around 7e-4 (with margin of 0.009)
@@ -106,3 +76,27 @@ Model described in paper (Dense + CNN):
- 0.343 on dev
- Why won't this train better :(
- Best validation loss was about 0.0011 (for margin of 0.009)
- After increasing the number of parameters/embedding size the model improved a bit
- Top 1 precision:
- 0.327 on test 1
- 0.302 on test 2
- MRR:
- 0.459 on test 1
- 0.433 on test 2
- Pre-trained the embedding layer via the EmbeddingModel, re-ran
- Top 1 precision:
- 0.472 on test 1
- 0.427 on test 2
- 0.470 on dev
- MRR:
- 0.595 on test 1
- 0.557 on test 2
- 0.596 on dev
1000 embed dims + 2000 CNN filters (total over 4 lengths):
- Loss ~6e-4 for margin of 0.009
- Test 1: 0.409 Top-1 Precision, 0.543 MRR
- Test 2: 0.376 Top-1 Precision, 0.507 MRR