mirror of
https://github.com/wassname/keras-language-modeling.git
synced 2026-09-12 12:33:54 +08:00
made everything make sense
This commit is contained in:
codekansas
@@ -155,8 +155,8 @@ class Evaluator:
|
||||
|
||||
if __name__ == '__main__':
|
||||
conf = {
|
||||
'question_len': 100,
|
||||
'answer_len': 100,
|
||||
'question_len': 10,
|
||||
'answer_len': 40,
|
||||
'n_words': 22353, # len(vocabulary) + 1
|
||||
'margin': 0.009,
|
||||
|
||||
@@ -184,13 +184,14 @@ if __name__ == '__main__':
|
||||
'mode': 'cosine',
|
||||
'gamma': 1,
|
||||
'c': 1,
|
||||
'd': 2,
|
||||
}
|
||||
}
|
||||
|
||||
evaluator = Evaluator(data_path, conf)
|
||||
|
||||
##### Define model ######
|
||||
model = EmbeddingModel(conf)
|
||||
model = RecurrentModel(conf)
|
||||
model.compile(optimizer='adam')
|
||||
|
||||
evaluator.train(model)
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
from __future__ import print_function
|
||||
|
||||
##############
|
||||
# Make model #
|
||||
##############
|
||||
import os
|
||||
|
||||
from keras.layers import Dropout, Embedding, Lambda, Convolution1D
|
||||
from keras.optimizers import RMSprop
|
||||
|
||||
from attention_lstm import AttentionLSTM
|
||||
|
||||
from keras.layers import Input, LSTM, merge
|
||||
from keras.models import Model
|
||||
import keras.backend as K
|
||||
|
||||
models_path = 'models/'
|
||||
|
||||
|
||||
def make_model(maxlen_question, maxlen_answer, n_words, n_embed_dims=128):
|
||||
n_lstm_dims = 141
|
||||
|
||||
# input
|
||||
question = Input(shape=(maxlen_question,), dtype='int32', name='question_input')
|
||||
answer_good = Input(shape=(maxlen_answer,), dtype='int32', name='answer_good_input')
|
||||
answer_bad = Input(shape=(maxlen_answer,), dtype='int32', name='answer_bad_input')
|
||||
|
||||
# language model
|
||||
embedding = Embedding(n_words, n_embed_dims)
|
||||
maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]))
|
||||
|
||||
# forward and backward lstms
|
||||
f_lstm = LSTM(n_lstm_dims, name='fq_lstm', consume_less='mem', return_sequences=True)
|
||||
b_lstm = LSTM(n_lstm_dims, name='bq_lstm', go_backwards=True, consume_less='mem', return_sequences=True)
|
||||
|
||||
# question part
|
||||
q_emb = embedding(question)
|
||||
q_emb = Dropout(0.25)(q_emb)
|
||||
q_fl = f_lstm(q_emb)
|
||||
q_bl = b_lstm(q_emb)
|
||||
q_out = merge([q_fl, q_bl], mode='concat', concat_axis=-1)
|
||||
q_out = maxpool(q_out)
|
||||
|
||||
# forward and backward attention lstms (paying attention to q_out)
|
||||
f_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, consume_less='mem', return_sequences=True)
|
||||
b_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, go_backwards=True, consume_less='mem', return_sequences=True)
|
||||
|
||||
# answer part
|
||||
ag_emb = embedding(answer_good)
|
||||
ag_emb = Dropout(0.25)(ag_emb)
|
||||
ag_fl = f_lstm_attention(ag_emb)
|
||||
ag_bl = b_lstm_attention(ag_emb)
|
||||
ag_out = merge([ag_fl, ag_bl], mode='concat', concat_axis=-1)
|
||||
ag_out = maxpool(ag_out)
|
||||
|
||||
ab_emb = embedding(answer_bad)
|
||||
ab_emb = Dropout(0.25)(ab_emb)
|
||||
ab_fl = f_lstm_attention(ab_emb)
|
||||
ab_bl = b_lstm_attention(ab_emb)
|
||||
ab_out = merge([ab_fl, ab_bl], mode='concat', concat_axis=-1)
|
||||
ab_out = maxpool(ab_out)
|
||||
|
||||
# merge together
|
||||
# note: `cos` refers to "cosine similarity", i.e. similar vectors should go to 1
|
||||
# for training's sake, "abs" limits range to between 0 and 1 (binary classification)
|
||||
good_out = merge([q_out, ag_out], name='good', mode='cos', dot_axes=1)
|
||||
bad_out = merge([q_out, ab_out], name='bad', mode='cos', dot_axes=1)
|
||||
|
||||
target = merge([good_out, bad_out], name='target', mode=lambda x: K.maximum(1e-6, 0.009 - x[0] + x[1]), output_shape=lambda x: x[0])
|
||||
|
||||
train_model = Model(input=[question, answer_good, answer_bad], output=target)
|
||||
test_model = Model(input=[question, answer_good], output=good_out)
|
||||
|
||||
print('Compiling model...')
|
||||
|
||||
optimizer = RMSprop(lr=0.0001, clipnorm=0.05)
|
||||
# optimizer = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True, clipgrad=0.1)
|
||||
|
||||
# this is more true to the paper: L = max{0, M - cosine(q, a+) + cosine(q, a-)}
|
||||
# below, "a" is a list of zeros and "b" is `target` above, i.e. 1 - cosine(q, a+) + cosine(q, a-)
|
||||
# loss = 'binary_crossentropy'
|
||||
# loss = 'mse'
|
||||
# loss = 'hinge'
|
||||
|
||||
def loss(y_true, y_pred):
|
||||
return y_pred
|
||||
|
||||
# unfortunately, the hinge loss approach means the "accuracy" metric isn't very valuable
|
||||
metrics = []
|
||||
|
||||
train_model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
|
||||
test_model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
|
||||
|
||||
return train_model, test_model
|
||||
|
||||
if __name__ == '__main__':
|
||||
# get the data set
|
||||
maxlen = 40 # words
|
||||
|
||||
from utils.get_data import get_data_set, create_dictionary_from_qas
|
||||
|
||||
dic = create_dictionary_from_qas()
|
||||
targets, questions, good_answers, bad_answers, n_dims = get_data_set(maxlen)
|
||||
|
||||
train_model, test_model = make_model(maxlen, n_dims)
|
||||
|
||||
print('Fitting model')
|
||||
train_model.fit([questions, good_answers, bad_answers], targets, nb_epoch=5, batch_size=128)
|
||||
train_model.save_weights(os.path.join(models_path, 'attention_lm_weights.h5'), overwrite=True)
|
||||
@@ -1,104 +0,0 @@
|
||||
from __future__ import print_function
|
||||
|
||||
##############
|
||||
# Make model #
|
||||
##############
|
||||
import os
|
||||
|
||||
from keras.layers import Dropout, Embedding, Lambda, Convolution1D, MaxPooling1D
|
||||
from keras.optimizers import RMSprop
|
||||
|
||||
from attention_lstm import AttentionLSTM
|
||||
|
||||
from keras.layers import Input, LSTM, merge
|
||||
from keras.models import Model
|
||||
import keras.backend as K
|
||||
|
||||
models_path = 'models/'
|
||||
|
||||
|
||||
def make_model(maxlen_question, maxlen_answer, n_words, n_embed_dims=128):
|
||||
n_lstm_dims = 400
|
||||
|
||||
# input
|
||||
question = Input(shape=(maxlen_question,), dtype='int32', name='question_input')
|
||||
answer_good = Input(shape=(maxlen_answer,), dtype='int32', name='answer_good_input')
|
||||
answer_bad = Input(shape=(maxlen_answer,), dtype='int32', name='answer_bad_input')
|
||||
|
||||
# language model
|
||||
embedding = Embedding(n_words, n_embed_dims, mask_zero=True)
|
||||
|
||||
# forward and backward lstms
|
||||
f_lstm = LSTM(n_lstm_dims, name='fq_lstm', consume_less='mem', return_sequences=True)
|
||||
|
||||
def maxpool_func(x, mask=None):
|
||||
return K.max(x, axis=1, keepdims=False)
|
||||
maxpool = Lambda(maxpool_func, output_shape=lambda x: (x[0], x[2]))
|
||||
|
||||
# question part
|
||||
q_emb = embedding(question)
|
||||
q_emb = Dropout(0.25)(q_emb)
|
||||
q_out = f_lstm(q_emb)
|
||||
q_out = maxpool(q_out)
|
||||
|
||||
# forward and backward attention lstms (paying attention to q_out)
|
||||
f_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, consume_less='mem', return_sequences=True)
|
||||
|
||||
# answer part
|
||||
ag_emb = embedding(answer_good)
|
||||
ag_emb = Dropout(0.25)(ag_emb)
|
||||
ag_out = f_lstm_attention(ag_emb)
|
||||
ag_out = maxpool(ag_out)
|
||||
|
||||
ab_emb = embedding(answer_bad)
|
||||
ab_emb = Dropout(0.25)(ab_emb)
|
||||
ab_out = f_lstm_attention(ab_emb)
|
||||
ab_out = maxpool(ab_out)
|
||||
|
||||
# merge together
|
||||
# note: `cos` refers to "cosine similarity", i.e. similar vectors should go to 1
|
||||
# for training's sake, "abs" limits range to between 0 and 1 (binary classification)
|
||||
good_out = merge([q_out, ag_out], name='good', mode='cos', dot_axes=1)
|
||||
bad_out = merge([q_out, ab_out], name='bad', mode='cos', dot_axes=1)
|
||||
|
||||
target = merge([good_out, bad_out], name='target', mode=lambda x: K.maximum(1e-6, 0.1 - x[0] + x[1]), output_shape=lambda x: x[0])
|
||||
|
||||
train_model = Model(input=[question, answer_good, answer_bad], output=target)
|
||||
test_model = Model(input=[question, answer_good], output=good_out)
|
||||
|
||||
print('Compiling model...')
|
||||
|
||||
optimizer = RMSprop(lr=0.0001, clipnorm=0.05)
|
||||
# optimizer = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True, clipgrad=0.1)
|
||||
|
||||
# this is more true to the paper: L = max{0, M - cosine(q, a+) + cosine(q, a-)}
|
||||
# below, "a" is a list of zeros and "b" is `target` above, i.e. 1 - cosine(q, a+) + cosine(q, a-)
|
||||
# loss = 'binary_crossentropy'
|
||||
# loss = 'mse'
|
||||
# loss = 'hinge'
|
||||
|
||||
def loss(y_true, y_pred):
|
||||
return y_pred
|
||||
|
||||
# unfortunately, the hinge loss approach means the "accuracy" metric isn't very valuable
|
||||
metrics = []
|
||||
|
||||
train_model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
|
||||
test_model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
|
||||
|
||||
return train_model, test_model
|
||||
|
||||
if __name__ == '__main__':
|
||||
# get the data set
|
||||
maxlen = 40 # words
|
||||
|
||||
from utils.get_data import get_data_set, create_dictionary_from_qas
|
||||
|
||||
dic = create_dictionary_from_qas()
|
||||
targets, questions, good_answers, bad_answers, n_dims = get_data_set(maxlen)
|
||||
|
||||
train_model, test_model = make_model(maxlen, n_dims)
|
||||
|
||||
print('Fitting model')
|
||||
train_model.fit([questions, good_answers, bad_answers], targets, nb_epoch=5, batch_size=128)
|
||||
train_model.save_weights(os.path.join(models_path, 'attention_lm_weights.h5'), overwrite=True)
|
||||
@@ -1,95 +0,0 @@
|
||||
from __future__ import print_function
|
||||
|
||||
##############
|
||||
# Make model #
|
||||
##############
|
||||
import os
|
||||
|
||||
from keras.engine import Merge
|
||||
from keras.layers import MaxPooling1D, Dropout, Embedding, Convolution1D, Dense, Lambda, TimeDistributed, \
|
||||
ActivityRegularization
|
||||
|
||||
models_path = 'models/'
|
||||
|
||||
|
||||
def make_model(maxlen_question, maxlen_answer, n_words, n_embed_dims=128):
|
||||
from keras.layers import Input, merge
|
||||
from keras.models import Model
|
||||
import keras.backend as K
|
||||
|
||||
# input
|
||||
question = Input(shape=(maxlen_question,), dtype='int32', name='question_input')
|
||||
answer_good = Input(shape=(maxlen_answer,), dtype='int32', name='answer_good_input')
|
||||
answer_bad = Input(shape=(maxlen_answer,), dtype='int32', name='answer_bad_input')
|
||||
|
||||
# language model
|
||||
embedding = Embedding(n_words, n_embed_dims)
|
||||
|
||||
# embedding
|
||||
q_emb = embedding(question)
|
||||
ag_emb = embedding(answer_good)
|
||||
ab_emb = embedding(answer_bad)
|
||||
|
||||
# dense
|
||||
dense = TimeDistributed(Dense(200, activation='tanh'))
|
||||
q_dense = dense(q_emb)
|
||||
ag_dense = dense(ag_emb)
|
||||
ab_dense = dense(ab_emb)
|
||||
|
||||
# regularlize
|
||||
q_dense = ActivityRegularization(l2=0.0001)(q_dense)
|
||||
ag_dense = ActivityRegularization(l2=0.0001)(ag_dense)
|
||||
ab_dense = ActivityRegularization(l2=0.0001)(ab_dense)
|
||||
|
||||
# dropout
|
||||
q_dense = Dropout(0.25)(q_dense)
|
||||
ag_dense = Dropout(0.25)(ag_dense)
|
||||
ab_dense = Dropout(0.25)(ab_dense)
|
||||
|
||||
# cnn
|
||||
cnns = [Convolution1D(filter_length=filt, nb_filter=1000, activation='relu', border_mode='same') for filt in [2, 3, 5, 7]]
|
||||
q_cnn = merge([cnn(q_dense) for cnn in cnns], mode='concat')
|
||||
ag_cnn = merge([cnn(ag_dense) for cnn in cnns], mode='concat')
|
||||
ab_cnn = merge([cnn(ab_dense) for cnn in cnns], mode='concat')
|
||||
|
||||
# dropout
|
||||
q_cnn = Dropout(0.25)(q_cnn)
|
||||
ag_cnn = Dropout(0.25)(ag_cnn)
|
||||
ab_cnn = Dropout(0.25)(ab_cnn)
|
||||
|
||||
# maxpooling
|
||||
# maxpool = MaxPooling1D(pool_length=2)
|
||||
maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]))
|
||||
q_pool = maxpool(q_cnn)
|
||||
ag_pool = maxpool(ag_cnn)
|
||||
ab_pool = maxpool(ab_cnn)
|
||||
|
||||
# tanh
|
||||
tanh = Lambda(lambda x: K.tanh(x))
|
||||
q_out = tanh(q_pool)
|
||||
ag_out = tanh(ag_pool)
|
||||
ab_out = tanh(ab_pool)
|
||||
|
||||
# merge together
|
||||
good_out = merge([q_out, ag_out], mode='cos', dot_axes=1)
|
||||
bad_out = merge([q_out, ab_out], mode='cos', dot_axes=1)
|
||||
target = merge([good_out, bad_out], name='target', mode=lambda x: K.maximum(1e-6, 0.009 - x[0] + x[1]), output_shape=lambda x: x[0])
|
||||
|
||||
train_model = Model(input=[question, answer_good, answer_bad], output=target)
|
||||
test_model = Model(input=[question, answer_good], output=good_out)
|
||||
|
||||
print('Compiling model...')
|
||||
|
||||
# optimizer = RMSprop(lr=0.01, clipnorm=0.05)
|
||||
optimizer = 'adam'
|
||||
|
||||
def loss(y_true, y_pred):
|
||||
return y_pred
|
||||
|
||||
# unfortunately, the hinge loss approach means the "accuracy" metric isn't very valuable
|
||||
metrics = []
|
||||
|
||||
train_model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
|
||||
test_model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
|
||||
|
||||
return train_model, test_model
|
||||
+6
-4
@@ -150,6 +150,8 @@ class LanguageModel:
|
||||
|
||||
|
||||
class EmbeddingModel(LanguageModel):
|
||||
''' This model actually performs stupidly well '''
|
||||
|
||||
def build(self):
|
||||
input, _ = self._get_inputs()
|
||||
|
||||
@@ -232,12 +234,12 @@ class RecurrentModel(LanguageModel):
|
||||
input_dropout = dropout(input_embedding)
|
||||
|
||||
# rnn
|
||||
# forward_lstm = LSTM(self.config.get('n_lstm_dims', 141), consume_less='mem', return_sequences=False)
|
||||
# backward_lstm = LSTM(self.config.get('n_lstm_dims', 141), consume_less='mem', return_sequences=False)
|
||||
# input_lstm = merge([forward_lstm(input_dropout), backward_lstm(input_dropout)], mode='concat', concat_axis=-1)
|
||||
forward_lstm = LSTM(self.config.get('n_lstm_dims', 141), consume_less='mem', return_sequences=True)
|
||||
backward_lstm = LSTM(self.config.get('n_lstm_dims', 141), consume_less='mem', return_sequences=True)
|
||||
input_lstm = merge([forward_lstm(input_dropout), backward_lstm(input_dropout)], mode='concat', concat_axis=-1)
|
||||
|
||||
# dropout
|
||||
# input_dropout = dropout(input_lstm)
|
||||
input_dropout = dropout(input_lstm)
|
||||
|
||||
# maxpooling
|
||||
maxpool = Lambda(lambda x: K.mean(K.exp(x), axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]))
|
||||
|
||||
@@ -49,4 +49,9 @@ Pure CNN Model:
|
||||
- Highest-rank answer: along with what Peggy and Steve mention make sure whoever you choose as your agent present you with multiple quote for disability insurance too often consumer think the cheap policy be good but it have likely that the cheap policy offer poor coverage your agent assume he or she do not represent just 1 company shall be able provide you different quote that reflect different type of coverage ( e.g. different benefit period , elimination period , different company , etc.
|
||||
- Rank of best answer: 347.0
|
||||
|
||||
Embedding + MaxPooling:
|
||||
- I can't believe this model performed so well. It blew the other ones out of the water, and trains ridiculously quickly.
|
||||
- Test 1: Top-1 Precision = 0.4933, MRR = 0.6189
|
||||
- Test 2: Top-1 Precision = 0.4606, MRR = 0.5968
|
||||
- Dev: Top-1 Precision = 0.4700, MRR = 0.6088
|
||||
|
||||
|
||||
Reference in New Issue
Block a user