fixed variable perf bug and added debug arg for testing on each epoch (#2)

Because of random word embedding for out of vocabulary words, the performance of the final saved model was variable. This is now fixed.

README.md

@@ -2,5 +2,5 @@
 
 Pytorch deep learning models.
 
-1. [Similarity Measure model (SM model)](./sm-model/README.md)
+1. [SM model](./sm-model/README.md): Similarity between question and candidate answers.
 

sm-model/.gitignore

@@ -0,0 +1,2 @@
+*pyc
+trec_eval-8.0/trec_eval

sm-model/main.py

@@ -38,7 +38,7 @@ def logargs(func):
 
 
 def compute_map_mrr(dataset_folder, set_folder, test_scores):
-    logger.info( "Running trec_eval script..." )
+    # logger.info( "Running trec_eval script..." )
     N = len(test_scores)
 
     qids_test, y_test = utils.get_test_qids_labels(dataset_folder, set_folder)
@@ -64,6 +64,7 @@ def compute_map_mrr(dataset_folder, set_folder, test_scores):
     pargs = shlex.split("/bin/sh run_eval.sh '{}'".format(args.dataset_folder))
     p = subprocess.Popen(pargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     pout, perr = p.communicate()    
+    
     lines = pout.split('\n')
     map = float(lines[0].strip().split()[-1])
     mrr = float(lines[1].strip().split()[-1])
@@ -87,6 +88,7 @@ if __name__ == "__main__":
     ap.add_argument('--filter_width', type=int, default=5)    
     ap.add_argument('--eta', help='Initial learning rate', default=0.001, type=float)
     ap.add_argument('--mom', help='SGD Momentum', default=0.0, type=float)    
+    ap.add_argument('--train_all', help='switches to train-all set', action="store_true")
     
     # epoch related arguments
     ap.add_argument('--epochs', type=int, default=25)
@@ -97,12 +99,17 @@ if __name__ == "__main__":
     ap.add_argument('--num_conv_filters', help="the number of convolution channels (lesser is faster)", default=100, type=int)
     ap.add_argument('--no_ext_feats', action="store_true", help="will not include external features in the model")    
     ap.add_argument('--no_loss_reg', help="no loss regularization", action="store_true")
+    ap.add_argument('--test_on_each_epoch', help='runs test on each epoch to track final performance', action="store_true")
 
     args = ap.parse_args()
 
     torch.manual_seed(1234)
     np.random.seed(1234)
 
+    train_set, dev_set, test_set = 'train', 'clean-dev', 'clean-test'
+    if args.train_all:
+        train_set, dev_set, test_set = 'train-all', 'raw-dev', 'raw-test'
+
     # cache word embeddings
     cache_file = os.path.splitext(args.word_vectors_file)[0] + '.cache'    
     utils.cache_word_embeddings(args.word_vectors_file, cache_file)
@@ -115,41 +122,48 @@ if __name__ == "__main__":
 
     torch.set_num_threads(args.num_threads)
     
-    trainer = Trainer(net, args.eta, args.mom, args.no_loss_reg)
+    trainer = Trainer(net, args.eta, args.mom, args.no_loss_reg, vec_dim)
     logger.info("Loading input data...")
-    trainer.load_input_data(args.dataset_folder, cache_file, 'train', 'clean-dev', 'clean-test')
+    trainer.load_input_data(args.dataset_folder, cache_file, train_set, dev_set, test_set)
 
     best_map = 0.0
     best_model = 0
-    
+
     for i in range(args.epochs):
         logger.info('------------- Training epoch {} --------------'.format(i+1))        
-        train_accuracy = trainer.train('train', args.batch_size, args.debugSingleBatch)        
+        train_accuracy = trainer.train(train_set, args.batch_size, args.debugSingleBatch)        
         if args.debugSingleBatch: sys.exit(0)
-        dev_scores = trainer.test('clean-dev', args.batch_size)
-
-        dev_map, dev_mrr = compute_map_mrr(args.dataset_folder, 'clean-dev', dev_scores)
+        
+        dev_scores = trainer.test(dev_set, args.batch_size)
+        
+        dev_map, dev_mrr = compute_map_mrr(args.dataset_folder, dev_set, dev_scores)
         logger.info("------- MAP {}, MRR {}".format(dev_map, dev_mrr))
 
         if dev_map - best_map > 1e-3: # new map is better than best map 
             best_model = i
             best_map = dev_map
+
             QAModel.save(net, args.dataset_folder, args.model_fname)
             logger.info('Achieved better dev_map ... saved model')
+
+        if args.test_on_each_epoch:            
+            test_scores = trainer.test(test_set, args.batch_size)            
+            map, mrr = compute_map_mrr(args.dataset_folder, test_set, test_scores)
+            logger.info("------- MAP {}, MRR {}".format(map, mrr))
         
         if (i - best_model) >= args.patience:
             logger.warning('No improvement since the last {} epochs. Stopping training'.format(i - best_model))
             break
     
-    logger.info(' ------------ Training epochs completed!')        
+    logger.info(' ------------ Training epochs completed! ------------')        
     logger.info('Best MAP in training phase = {:.4f}'.format(best_map))
 
-    model = QAModel.load(args.dataset_folder, args.model_fname)
-    evaluator = Trainer(model, args.eta, args.mom, args.no_loss_reg)
-    evaluator.load_input_data(args.dataset_folder, cache_file, None, None, 'clean-test')
-    test_scores = evaluator.test('clean-test', args.batch_size)
+    trained_model = QAModel.load(args.dataset_folder, args.model_fname)
+    evaluator = Trainer(trained_model, args.eta, args.mom, args.no_loss_reg, vec_dim)    
+    evaluator.load_input_data(args.dataset_folder, cache_file, None, None, test_set)
+    test_scores = evaluator.test(test_set, args.batch_size)
     
-    map, mrr = compute_map_mrr(args.dataset_folder, 'clean-test', test_scores)
+    map, mrr = compute_map_mrr(args.dataset_folder, test_set, test_scores)
     logger.info("------- MAP {}, MRR {}".format(map, mrr))
 
     

sm-model/model.py

@@ -73,7 +73,7 @@ class QAModel(nn.Module):
             x = torch.cat([q, a, ext_feats], 1)        
             # logger.debug('with ext_feats')
         
-        logger.debug('featvec x: {}'.format(x))
+        # logger.debug('featvec x: {}'.format(x))
         # logger.debug(x.creator)
 
         x = self.combined_feature_vector.forward(x)

sm-model/overlap_features.py

@@ -115,6 +115,7 @@ def compute_dfs(docs):
 if __name__ == '__main__':
   ap = argparse.ArgumentParser(description="compute overlap features for SM model")
   ap.add_argument("dataset", help="path/to/dataset-directory", default="../../data/TrecQA")
+  ap.add_argument("--train_all", help="will generate overlap features for the train-all dataset", action="store_true")
   args = ap.parse_args()
 
   stoplist = set([line.strip() for line in open('stopwords.txt')])
@@ -129,6 +130,9 @@ if __name__ == '__main__':
   # sub_dirs = ['train/', 'raw-dev/', 'test.minimal/','test.complete/']
   # sub_dirs = ['train-all/', 'raw-dev/', 'raw-test/']
   sub_dirs = ['train/', 'clean-dev/', 'clean-test/']
+  if args.train_all:
+      sub_dirs = ['train-all/', 'raw-dev/', 'raw-test/']
+      
   for sub in sub_dirs:
     qids, questions, answers, labels = load_data(base_dir+sub)
     all_questions.extend(questions)

sm-model/train.py

@@ -27,7 +27,13 @@ logger.addHandler(ch)
 
 class Trainer(object):
     
-    def __init__(self, model, eta, mom, no_loss_reg):
+    def __init__(self, model, eta, mom, no_loss_reg, vec_dim):
+        # set the random seeds for every instance of trainer. 
+        # needed to ensure reproduction of random word vectors for out of vocab terms
+        torch.manual_seed(1234)
+        np.random.seed(1234)
+        self.unk_term = np.random.uniform(-0.25, 0.25, vec_dim) 
+
         self.reg = 1e-5
         self.no_loss_reg = no_loss_reg
         self.model = model
@@ -37,13 +43,17 @@ class Trainer(object):
 
         self.datasets = {}
         self.embeddings = {}
+        self.vec_dim = vec_dim
+        
 
     def load_input_data(self, dataset_root_folder, word_vectors_cache_file, train_set_folder, dev_set_folder, test_set_folder):
-        for set_folder in [train_set_folder, dev_set_folder, test_set_folder]:
+        for set_folder in [test_set_folder, dev_set_folder, train_set_folder]:
             if set_folder:
                 self.datasets[set_folder] = utils.read_in_dataset(dataset_root_folder, set_folder)
                 # NOTE: self.datasets[set_folder] = questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats 
-                self.embeddings[set_folder] = utils.load_cached_embeddings(word_vectors_cache_file, self.datasets[set_folder][3])
+                self.embeddings[set_folder] = utils.load_cached_embeddings(word_vectors_cache_file, 
+                    self.datasets[set_folder][3], [] if "train" in set_folder else self.unk_term)
+            
         
     def regularize_loss(self, loss):       
         
@@ -71,8 +81,8 @@ class Trainer(object):
         # logger.debug('loss after criterion {}'.format(loss))
 
         # NOTE: regularizing location 1
-        # if not self.no_loss_reg:
-        #     loss = self.regularize_loss(loss)
+        if not self.no_loss_reg:
+             loss = self.regularize_loss(loss)
         #     logger.debug('loss after regularizing {}'.format(loss))
         
         loss.backward()
@@ -82,8 +92,8 @@ class Trainer(object):
         # logger.debug('params grads {}'.format([p.grad for p in self.model.parameters()]))
        
         # NOTE: regularizing location 2. It would seem that location 1 is correct?
-        if not self.no_loss_reg:
-            loss = self.regularize_loss(loss)
+        #if not self.no_loss_reg:
+        #    loss = self.regularize_loss(loss)
             # logger.debug('loss after regularizing {}'.format(loss))
 
         self.optimizer.step()
@@ -95,14 +105,9 @@ class Trainer(object):
         return loss.data[0], self.pred_equals_y(output, ys)
 
 
-    def pred_equals_y(self, pred, y):
-        # logger.debug('pred_equals_y:')
-        # logger.debug(pred)
-        # logger.debug(y)
-        _, best = pred.max(1)
-        # logger.debug('{} {}'.format(_, best))
-        best = best.data.long().squeeze()
-        # logger.debug(best)
+    def pred_equals_y(self, pred, y):        
+        _, best = pred.max(1)        
+        best = best.data.long().squeeze()        
         return torch.sum(y.data.long() == best)
 
 
@@ -110,7 +115,7 @@ class Trainer(object):
         logger.info('----- Predictions on {} '.format(set_folder))
                 
         questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = self.datasets[set_folder]
-        word_vectors, vec_dim = self.embeddings[set_folder]
+        word_vectors, vec_dim = self.embeddings[set_folder], self.vec_dim
         
         self.model.eval()
 
@@ -159,7 +164,7 @@ class Trainer(object):
         train_start_time = time.time()
 
         questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = self.datasets[set_folder]
-        word_vectors, vec_dim = self.embeddings[set_folder]
+        word_vectors, vec_dim = self.embeddings[set_folder], self.vec_dim
 
         # set model for training modep
         self.model.train()

sm-model/utils.py

@@ -105,9 +105,9 @@ def load_embedding_dimensions(cache_file):
     return vocab_size, vec_dim
 
     
-def load_cached_embeddings(cache_file, vocab_list):
-    logger.debug( 'loading cached embeddings ')
-    w2v_dict = {}
+def load_cached_embeddings(cache_file, vocab_list, oov_vec = []):
+    logger.debug( 'loading cached embeddings ')    
+
     with open(cache_file + '.dimensions') as d:
         vocab_size, vec_dim = [int(e) for e in d.read().strip().split()]
 
@@ -120,12 +120,16 @@ def load_cached_embeddings(cache_file, vocab_list):
     vocab_dict = {w:k for k,w in enumerate(w2v_vocab_list)}
     
     # Read w2v for vocab appears in Q and A
+    w2v_dict = {}
     for word in vocab_list:
+        if word in w2v_dict:
+            continue
         if word in vocab_dict:
             w2v_dict[word] = W[vocab_dict[word]]
         else:
-            w2v_dict[word] = np.random.uniform(-0.25, 0.25, vec_dim) 
-    return w2v_dict, vec_dim
+            w2v_dict[word] =  np.random.uniform(-0.25, 0.25, vec_dim) if len(oov_vec) == 0 else oov_vec
+            #w2v_dict[word] = W[vocab_dict["unk"]]
+    return w2v_dict
 
 
 def read_in_dataset(dataset_folder, set_folder):
@@ -153,4 +157,15 @@ def get_test_qids_labels(dataset_folder, set_folder):
     set_path = os.path.join(dataset_folder, set_folder)
     qids = [ line.strip() for line in open(os.path.join(set_path, 'id.txt')).readlines() ]
     labels = np.array([ int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines() ])
-    return qids, labels
+    return qids, labels
+
+
+if __name__ == "__main__":
+    
+    vocab = [ "unk", "idontreallythinkthiswordexists", "hello" ]
+
+    w2v_dict, vec_dim =  load_cached_embeddings("../../data/word2vec-models/aquaint+wiki.txt.gz.ndim=50.cache", vocab)
+    
+    for w, v in w2v_dict.iteritems():
+        print w
+        print v