From f91f1055683ed23da19b6fef46006bc084e3c5be Mon Sep 17 00:00:00 2001 From: gauravbaruah Date: Thu, 30 Mar 2017 12:33:20 -0400 Subject: [PATCH] fixed variable perf bug and added debug arg for testing on each epoch (#2) Because of random word embedding for out of vocabulary words, the performance of the final saved model was variable. This is now fixed. --- README.md | 2 +- sm-model/.gitignore | 2 ++ sm-model/main.py | 42 ++++++++++++++++++++++++------------ sm-model/model.py | 2 +- sm-model/overlap_features.py | 4 ++++ sm-model/train.py | 39 ++++++++++++++++++--------------- sm-model/utils.py | 27 +++++++++++++++++------ 7 files changed, 79 insertions(+), 39 deletions(-) create mode 100644 sm-model/.gitignore diff --git a/README.md b/README.md index 7f5ef6b..ae884ad 100644 --- a/README.md +++ b/README.md @@ -2,5 +2,5 @@ Pytorch deep learning models. -1. [Similarity Measure model (SM model)](./sm-model/README.md) +1. [SM model](./sm-model/README.md): Similarity between question and candidate answers. diff --git a/sm-model/.gitignore b/sm-model/.gitignore new file mode 100644 index 0000000..8206f0d --- /dev/null +++ b/sm-model/.gitignore @@ -0,0 +1,2 @@ +*pyc +trec_eval-8.0/trec_eval \ No newline at end of file diff --git a/sm-model/main.py b/sm-model/main.py index 1aa6d91..52e69e2 100644 --- a/sm-model/main.py +++ b/sm-model/main.py @@ -38,7 +38,7 @@ def logargs(func): def compute_map_mrr(dataset_folder, set_folder, test_scores): - logger.info( "Running trec_eval script..." ) + # logger.info( "Running trec_eval script..." ) N = len(test_scores) qids_test, y_test = utils.get_test_qids_labels(dataset_folder, set_folder) @@ -64,6 +64,7 @@ def compute_map_mrr(dataset_folder, set_folder, test_scores): pargs = shlex.split("/bin/sh run_eval.sh '{}'".format(args.dataset_folder)) p = subprocess.Popen(pargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE) pout, perr = p.communicate() + lines = pout.split('\n') map = float(lines[0].strip().split()[-1]) mrr = float(lines[1].strip().split()[-1]) @@ -87,6 +88,7 @@ if __name__ == "__main__": ap.add_argument('--filter_width', type=int, default=5) ap.add_argument('--eta', help='Initial learning rate', default=0.001, type=float) ap.add_argument('--mom', help='SGD Momentum', default=0.0, type=float) + ap.add_argument('--train_all', help='switches to train-all set', action="store_true") # epoch related arguments ap.add_argument('--epochs', type=int, default=25) @@ -97,12 +99,17 @@ if __name__ == "__main__": ap.add_argument('--num_conv_filters', help="the number of convolution channels (lesser is faster)", default=100, type=int) ap.add_argument('--no_ext_feats', action="store_true", help="will not include external features in the model") ap.add_argument('--no_loss_reg', help="no loss regularization", action="store_true") + ap.add_argument('--test_on_each_epoch', help='runs test on each epoch to track final performance', action="store_true") args = ap.parse_args() torch.manual_seed(1234) np.random.seed(1234) + train_set, dev_set, test_set = 'train', 'clean-dev', 'clean-test' + if args.train_all: + train_set, dev_set, test_set = 'train-all', 'raw-dev', 'raw-test' + # cache word embeddings cache_file = os.path.splitext(args.word_vectors_file)[0] + '.cache' utils.cache_word_embeddings(args.word_vectors_file, cache_file) @@ -115,41 +122,48 @@ if __name__ == "__main__": torch.set_num_threads(args.num_threads) - trainer = Trainer(net, args.eta, args.mom, args.no_loss_reg) + trainer = Trainer(net, args.eta, args.mom, args.no_loss_reg, vec_dim) logger.info("Loading input data...") - trainer.load_input_data(args.dataset_folder, cache_file, 'train', 'clean-dev', 'clean-test') + trainer.load_input_data(args.dataset_folder, cache_file, train_set, dev_set, test_set) best_map = 0.0 best_model = 0 - + for i in range(args.epochs): logger.info('------------- Training epoch {} --------------'.format(i+1)) - train_accuracy = trainer.train('train', args.batch_size, args.debugSingleBatch) + train_accuracy = trainer.train(train_set, args.batch_size, args.debugSingleBatch) if args.debugSingleBatch: sys.exit(0) - dev_scores = trainer.test('clean-dev', args.batch_size) - - dev_map, dev_mrr = compute_map_mrr(args.dataset_folder, 'clean-dev', dev_scores) + + dev_scores = trainer.test(dev_set, args.batch_size) + + dev_map, dev_mrr = compute_map_mrr(args.dataset_folder, dev_set, dev_scores) logger.info("------- MAP {}, MRR {}".format(dev_map, dev_mrr)) if dev_map - best_map > 1e-3: # new map is better than best map best_model = i best_map = dev_map + QAModel.save(net, args.dataset_folder, args.model_fname) logger.info('Achieved better dev_map ... saved model') + + if args.test_on_each_epoch: + test_scores = trainer.test(test_set, args.batch_size) + map, mrr = compute_map_mrr(args.dataset_folder, test_set, test_scores) + logger.info("------- MAP {}, MRR {}".format(map, mrr)) if (i - best_model) >= args.patience: logger.warning('No improvement since the last {} epochs. Stopping training'.format(i - best_model)) break - logger.info(' ------------ Training epochs completed!') + logger.info(' ------------ Training epochs completed! ------------') logger.info('Best MAP in training phase = {:.4f}'.format(best_map)) - model = QAModel.load(args.dataset_folder, args.model_fname) - evaluator = Trainer(model, args.eta, args.mom, args.no_loss_reg) - evaluator.load_input_data(args.dataset_folder, cache_file, None, None, 'clean-test') - test_scores = evaluator.test('clean-test', args.batch_size) + trained_model = QAModel.load(args.dataset_folder, args.model_fname) + evaluator = Trainer(trained_model, args.eta, args.mom, args.no_loss_reg, vec_dim) + evaluator.load_input_data(args.dataset_folder, cache_file, None, None, test_set) + test_scores = evaluator.test(test_set, args.batch_size) - map, mrr = compute_map_mrr(args.dataset_folder, 'clean-test', test_scores) + map, mrr = compute_map_mrr(args.dataset_folder, test_set, test_scores) logger.info("------- MAP {}, MRR {}".format(map, mrr)) diff --git a/sm-model/model.py b/sm-model/model.py index a8ab5aa..72fcfcf 100644 --- a/sm-model/model.py +++ b/sm-model/model.py @@ -73,7 +73,7 @@ class QAModel(nn.Module): x = torch.cat([q, a, ext_feats], 1) # logger.debug('with ext_feats') - logger.debug('featvec x: {}'.format(x)) + # logger.debug('featvec x: {}'.format(x)) # logger.debug(x.creator) x = self.combined_feature_vector.forward(x) diff --git a/sm-model/overlap_features.py b/sm-model/overlap_features.py index 6374d7b..97b677f 100644 --- a/sm-model/overlap_features.py +++ b/sm-model/overlap_features.py @@ -115,6 +115,7 @@ def compute_dfs(docs): if __name__ == '__main__': ap = argparse.ArgumentParser(description="compute overlap features for SM model") ap.add_argument("dataset", help="path/to/dataset-directory", default="../../data/TrecQA") + ap.add_argument("--train_all", help="will generate overlap features for the train-all dataset", action="store_true") args = ap.parse_args() stoplist = set([line.strip() for line in open('stopwords.txt')]) @@ -129,6 +130,9 @@ if __name__ == '__main__': # sub_dirs = ['train/', 'raw-dev/', 'test.minimal/','test.complete/'] # sub_dirs = ['train-all/', 'raw-dev/', 'raw-test/'] sub_dirs = ['train/', 'clean-dev/', 'clean-test/'] + if args.train_all: + sub_dirs = ['train-all/', 'raw-dev/', 'raw-test/'] + for sub in sub_dirs: qids, questions, answers, labels = load_data(base_dir+sub) all_questions.extend(questions) diff --git a/sm-model/train.py b/sm-model/train.py index 4c46ee4..19214d3 100644 --- a/sm-model/train.py +++ b/sm-model/train.py @@ -27,7 +27,13 @@ logger.addHandler(ch) class Trainer(object): - def __init__(self, model, eta, mom, no_loss_reg): + def __init__(self, model, eta, mom, no_loss_reg, vec_dim): + # set the random seeds for every instance of trainer. + # needed to ensure reproduction of random word vectors for out of vocab terms + torch.manual_seed(1234) + np.random.seed(1234) + self.unk_term = np.random.uniform(-0.25, 0.25, vec_dim) + self.reg = 1e-5 self.no_loss_reg = no_loss_reg self.model = model @@ -37,13 +43,17 @@ class Trainer(object): self.datasets = {} self.embeddings = {} + self.vec_dim = vec_dim + def load_input_data(self, dataset_root_folder, word_vectors_cache_file, train_set_folder, dev_set_folder, test_set_folder): - for set_folder in [train_set_folder, dev_set_folder, test_set_folder]: + for set_folder in [test_set_folder, dev_set_folder, train_set_folder]: if set_folder: self.datasets[set_folder] = utils.read_in_dataset(dataset_root_folder, set_folder) # NOTE: self.datasets[set_folder] = questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats - self.embeddings[set_folder] = utils.load_cached_embeddings(word_vectors_cache_file, self.datasets[set_folder][3]) + self.embeddings[set_folder] = utils.load_cached_embeddings(word_vectors_cache_file, + self.datasets[set_folder][3], [] if "train" in set_folder else self.unk_term) + def regularize_loss(self, loss): @@ -71,8 +81,8 @@ class Trainer(object): # logger.debug('loss after criterion {}'.format(loss)) # NOTE: regularizing location 1 - # if not self.no_loss_reg: - # loss = self.regularize_loss(loss) + if not self.no_loss_reg: + loss = self.regularize_loss(loss) # logger.debug('loss after regularizing {}'.format(loss)) loss.backward() @@ -82,8 +92,8 @@ class Trainer(object): # logger.debug('params grads {}'.format([p.grad for p in self.model.parameters()])) # NOTE: regularizing location 2. It would seem that location 1 is correct? - if not self.no_loss_reg: - loss = self.regularize_loss(loss) + #if not self.no_loss_reg: + # loss = self.regularize_loss(loss) # logger.debug('loss after regularizing {}'.format(loss)) self.optimizer.step() @@ -95,14 +105,9 @@ class Trainer(object): return loss.data[0], self.pred_equals_y(output, ys) - def pred_equals_y(self, pred, y): - # logger.debug('pred_equals_y:') - # logger.debug(pred) - # logger.debug(y) - _, best = pred.max(1) - # logger.debug('{} {}'.format(_, best)) - best = best.data.long().squeeze() - # logger.debug(best) + def pred_equals_y(self, pred, y): + _, best = pred.max(1) + best = best.data.long().squeeze() return torch.sum(y.data.long() == best) @@ -110,7 +115,7 @@ class Trainer(object): logger.info('----- Predictions on {} '.format(set_folder)) questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = self.datasets[set_folder] - word_vectors, vec_dim = self.embeddings[set_folder] + word_vectors, vec_dim = self.embeddings[set_folder], self.vec_dim self.model.eval() @@ -159,7 +164,7 @@ class Trainer(object): train_start_time = time.time() questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = self.datasets[set_folder] - word_vectors, vec_dim = self.embeddings[set_folder] + word_vectors, vec_dim = self.embeddings[set_folder], self.vec_dim # set model for training modep self.model.train() diff --git a/sm-model/utils.py b/sm-model/utils.py index 246cd01..44e6056 100644 --- a/sm-model/utils.py +++ b/sm-model/utils.py @@ -105,9 +105,9 @@ def load_embedding_dimensions(cache_file): return vocab_size, vec_dim -def load_cached_embeddings(cache_file, vocab_list): - logger.debug( 'loading cached embeddings ') - w2v_dict = {} +def load_cached_embeddings(cache_file, vocab_list, oov_vec = []): + logger.debug( 'loading cached embeddings ') + with open(cache_file + '.dimensions') as d: vocab_size, vec_dim = [int(e) for e in d.read().strip().split()] @@ -120,12 +120,16 @@ def load_cached_embeddings(cache_file, vocab_list): vocab_dict = {w:k for k,w in enumerate(w2v_vocab_list)} # Read w2v for vocab appears in Q and A + w2v_dict = {} for word in vocab_list: + if word in w2v_dict: + continue if word in vocab_dict: w2v_dict[word] = W[vocab_dict[word]] else: - w2v_dict[word] = np.random.uniform(-0.25, 0.25, vec_dim) - return w2v_dict, vec_dim + w2v_dict[word] = np.random.uniform(-0.25, 0.25, vec_dim) if len(oov_vec) == 0 else oov_vec + #w2v_dict[word] = W[vocab_dict["unk"]] + return w2v_dict def read_in_dataset(dataset_folder, set_folder): @@ -153,4 +157,15 @@ def get_test_qids_labels(dataset_folder, set_folder): set_path = os.path.join(dataset_folder, set_folder) qids = [ line.strip() for line in open(os.path.join(set_path, 'id.txt')).readlines() ] labels = np.array([ int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines() ]) - return qids, labels \ No newline at end of file + return qids, labels + + +if __name__ == "__main__": + + vocab = [ "unk", "idontreallythinkthiswordexists", "hello" ] + + w2v_dict, vec_dim = load_cached_embeddings("../../data/word2vec-models/aquaint+wiki.txt.gz.ndim=50.cache", vocab) + + for w, v in w2v_dict.iteritems(): + print w + print v \ No newline at end of file