Castorini smmodel now python 3 compatible. (#5)

Model is now python 3.6 compatible.

README.md

@@ -9,7 +9,8 @@ Pytorch deep learning models.
 
 You need Python 3.6 to use the models in this repository.
 
-As per [pytorch.org](pytorch.org) "Anaconda is our recommended package manager"
+As per [pytorch.org](pytorch.org) 
+> [Anaconda](https://www.continuum.io/downloads) is our recommended package manager
 
 ```conda install pytorch torchvision -c soumith```
 
@@ -20,6 +21,12 @@ We also recommend [gensim](https://radimrehurek.com/gensim/). We use some gensim
 ```conda install gensim```
 
 
+Pytorch has good support for GPU computations. 
+CUDA installation guide for linux can be found [here](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/)
+
+**NOTE**: Install CUDA libraries **before** installing conda and pytorch.
+
+
 ## Castor-data
 
 Sourcing and pre-processing of input data for each model is described in respective ```model/README.md```'s

sm-model/README.md

@@ -8,6 +8,9 @@
 1. figure out if the L2 regularization is correct
 2. Batch size of 50 (current batch_size = 1)
 
+#### Getting the data
+
+TODO:
 
 #### Running it
 

sm-model/SMCastorini.py

@@ -1,12 +1,12 @@
-import torch
-from torch.autograd import Variable
-import numpy as np
 import os
-from nltk.tokenize import TreebankWordTokenizer
-from nltk.stem.porter import *
-from collections import defaultdict
 import pickle
 import string
+from collections import defaultdict
+
+import numpy as np
+import torch
+from nltk.tokenize import TreebankWordTokenizer
+from torch.autograd import Variable
 
 from model import QAModel
 
@@ -29,7 +29,8 @@ class SMModelCastorini(object):
 
         # word dfs
         if os.path.isfile(word2dfs_file):
-		    self.word2dfs = pickle.load( open( "word2dfs.p", "rb" ) )
+            with open(word2dfs_file, "rb") as w2dfin:
+                self.word2dfs = pickle.load(w2dfin)
 
 
     def _preload_cached_embeddings(self, cache_file):
@@ -47,7 +48,7 @@ class SMModelCastorini(object):
 
 
     def parser(self, q, a):
-    	q_toks = TreebankWordTokenizer().tokenize(q)
+        q_toks = TreebankWordTokenizer().tokenize(q)
         q_str = ' '.join(q_toks).lower()
         a_list = []
         for ans in a:
@@ -58,7 +59,7 @@ class SMModelCastorini(object):
 
 
     def compute_overlap_features(self, q_str, a_list, word2df=None, stoplist=None):
-    	word2df = word2df if word2df else {}
+        word2df = word2df if word2df else {}
         stoplist = stoplist if stoplist else set()
         feats_overlap = []
         for a in a_list:
@@ -105,24 +106,20 @@ class SMModelCastorini(object):
             else:
                 emb = torch.from_numpy(self.W[self.vocab_dict[word]])                        
             word_embeddings[i] = emb            
-
         input_tensor = torch.zeros(1, self.vec_dim, len(terms))
-        input_tensor[0] = torch.transpose(word_embeddings, 0 , 1)
+        input_tensor[0] = torch.transpose(word_embeddings, 0, 1)
         return input_tensor
 
 
-    def get_tensorized_inputs(self, batch_ques, batch_sents, batch_ext_feats):
-        
-        assert(1 == len(batch_ques) )
-        
+    def get_tensorized_inputs(self, batch_ques, batch_sents, batch_ext_feats):     
+        assert(1 == len(batch_ques))        
         tensorized_inputs = []
         for i in range(len(batch_ques)):
-            xq = Variable(self.make_input_matrix(batch_ques[i]) ) 
-            xs = Variable(self.make_input_matrix(batch_sents[i]) )
+            xq = Variable(self.make_input_matrix(batch_ques[i])) 
+            xs = Variable(self.make_input_matrix(batch_sents[i]))
             ext_feats = Variable(torch.FloatTensor(batch_ext_feats[i]))
             ext_feats =torch.unsqueeze(ext_feats, 0)            
             tensorized_inputs.append((xq, xs, ext_feats))
-
         return tensorized_inputs
 
 
@@ -141,32 +138,33 @@ class SMModelCastorini(object):
             xq, xa, x_ext_feats = self.get_tensorized_inputs([q], [a_list[i]], [overlap_feats_vec[i]])[0]          
             pred = self.model(xq, xa, x_ext_feats)               
             pred = torch.exp(pred)
-            scores_sentences.append( (pred.data.squeeze()[1], a_list[i]) )
+            scores_sentences.append((pred.data.squeeze()[1], a_list[i]))
         
         return scores_sentences
 
 
 if __name__ == "__main__":
     
-
-    smmodel = SMModelCastorini('../../data/TrecQA/sm.model.aquaint.castorini', 
+    smmodel = SMModelCastorini('sm.model.py3', 
                     '../../data/word2vec-models/aquaint+wiki.txt.gz.ndim=50.cache',
                     'stopwords.txt',
                     'word2dfs.p')
-    
-    
+        
     q = "who is the author of the book , `` the iron lady : a biography of margaret thatcher '' ?"
     a = [ 
-    "the iron lady ; a biography of margaret thatcher by hugo young -lrb- farrar , straus & giroux -rrb-",
-    "in this same revisionist mold , hugo young , the distinguished british journalist , has performed a brilliant dissection of the notion of thatcher as a conservative icon .",
-    "in `` the iron lady , '' young traces the winding staircase of fortune that transformed the younger daughter of a provincial english grocer into the greatest woman political leader since catherine the great .",
-    "`` he is the very essence of the classless meritocrat , '' says hugo young , thatcher 's biographer .",
-    "from her father , young argues , she inherited a `` joyless earnestness '' that combined with her early interest in science to produce the roots of her public character .",
-    "this is not the answer",
-    "asdfawe asdf sertse dgfsgsfg"
+        "the iron lady ; a biography of margaret thatcher by hugo young -lrb- farrar , straus & giroux -rrb-",
+        "in this same revisionist mold , hugo young , the distinguished british journalist , has performed a brilliant \
+         dissection of the notion of thatcher as a conservative icon .",
+        "in `` the iron lady , '' young traces the winding staircase of fortune that transformed the younger daughter \
+        of a provincial english grocer into the greatest woman political leader since catherine the great .",
+        "`` he is the very essence of the classless meritocrat , '' says hugo young , thatcher 's biographer .",
+        "from her father , young argues , she inherited a `` joyless earnestness '' that combined with her early \
+        interest in science to produce the roots of her public character .",
+        "this is not the answer",
+        "asdfawe asdf sertse dgfsgsfg"
     ]
 	
     ss = smmodel.rerank_candidate_answers(q, a)
     print('Question:', q)
     for score, sentence in ss:
-        print(score, '\t', sentence)
+        print(score, '\t', sentence)

sm-model/main.py

@@ -1,22 +1,19 @@
-import os 
-import sys
-import time
-import glob
 import argparse
-import numpy as np
 
-import pandas as pd
-import subprocess
+import os
 import shlex
+import subprocess
+import sys
 
+import numpy as np
+import pandas as pd
 import torch
-import torch.optim as optim
 import torch.nn as nn
+import torch.optim as optim
 from torch.autograd import Variable
 
-
-from model import QAModel
 import utils
+from model import QAModel
 from train import Trainer
 
 # logging setup
@@ -38,7 +35,7 @@ def logargs(func):
 
 
 def compute_map_mrr(dataset_folder, set_folder, test_scores):
-    # logger.info( "Running trec_eval script..." )
+    # logger.info("Running trec_eval script...")
     N = len(test_scores)
 
     qids_test, y_test = utils.get_test_qids_labels(dataset_folder, set_folder)
@@ -65,7 +62,7 @@ def compute_map_mrr(dataset_folder, set_folder, test_scores):
     p = subprocess.Popen(pargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     pout, perr = p.communicate()    
     
-    lines = pout.split('\n')
+    lines = pout.split(b'\n')
     map = float(lines[0].strip().split()[-1])
     mrr = float(lines[1].strip().split()[-1])
     return map, mrr
@@ -165,5 +162,3 @@ if __name__ == "__main__":
     
     map, mrr = compute_map_mrr(args.dataset_folder, test_set, test_scores)
     logger.info("------- MAP {}, MRR {}".format(map, mrr))
-
-    

sm-model/model.py

@@ -1,10 +1,10 @@
+import os
+
+import numpy as np
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
 from torch.autograd import Variable
-import numpy as np
-
-import os
 
 # logging setup
 import logging
@@ -23,11 +23,13 @@ class QAModel(nn.Module):
     @staticmethod
     def save(model, out_folder, model_fname):
         torch.save(model, os.path.join(out_folder, model_fname))
+        
 
     @staticmethod
     def load(in_folder, model_fname):
         return torch.load(os.path.join(in_folder, model_fname))
-
+        
+        
     def __init__(self, input_n_dim, filter_width, conv_filters=100, no_ext_feats=False, ext_feats_size=4, n_classes=2):
         super(QAModel, self).__init__()
 

sm-model/overlap_features.py

@@ -169,7 +169,7 @@ if __name__ == '__main__':
     scaler = StandardScaler()
     print("Scaling overlap features")
     overlap_feats = scaler.fit_transform(overlap_feats)
-    print(overlap_feats[:3]    )
+    print(overlap_feats[:3])
 
     '''q_overlap_indices, a_overlap_indices = compute_overlap_idx(questions, answers, stoplist, q_max_sent_length, a_max_sent_length)
     print(q_overlap_indices[:3], a_overlap_indices[:3])

sm-model/train.py

@@ -1,14 +1,11 @@
-import os 
-import sys
+import argparse
 
 import time
-import glob
-import argparse
-import numpy as np
 
+import numpy as np
 import torch
-import torch.optim as optim
 import torch.nn as nn
+import torch.optim as optim
 from torch.autograd import Variable
 
 import utils
@@ -39,7 +36,7 @@ class Trainer(object):
         self.model = model
         self.criterion = nn.CrossEntropyLoss()
         #self.criterion = nn.NLLLoss()
-        self.optimizer = optim.SGD(self.model.parameters(), lr=eta, momentum=mom, weight_decay=(0 if no_loss_reg else self.reg) ) 
+        self.optimizer = optim.SGD(self.model.parameters(), lr=eta, momentum=mom, weight_decay=(0 if no_loss_reg else self.reg)) 
 
         self.datasets = {}
         self.embeddings = {}
@@ -123,7 +120,7 @@ class Trainer(object):
 
         total_loss = 0.0
         total_correct = 0.0
-        num_batches = np.ceil(len(questions)/batch_size )
+        num_batches = np.ceil(len(questions)/batch_size)
         y_pred = np.zeros(len(questions))
         ypc = 0
             
@@ -154,8 +151,8 @@ class Trainer(object):
         # logger.info('{}_correct {}'.format(set_folder, total_correct))
         # logger.info('{}_loss {}'.format(set_folder, total_loss.data[0]))
         logger.info('{} total {}'.format(set_folder, len(labels)))
-        # logger.info('{}_loss = {:.4f}, acc = {:.4f}'.format( set_folder, total_loss.data[0]/len(labels), float(total_correct)/len(labels) ))
-        #logger.info('{}_loss = {:.4f}'.format( set_folder, total_loss.data[0]/len(labels) ))
+        # logger.info('{}_loss = {:.4f}, acc = {:.4f}'.format(set_folder, total_loss.data[0]/len(labels), float(total_correct)/len(labels))
+        #logger.info('{}_loss = {:.4f}'.format(set_folder, total_loss.data[0]/len(labels)))
 
         return y_pred
 
@@ -170,7 +167,7 @@ class Trainer(object):
         self.model.train()
 
         train_loss, train_correct = 0., 0.
-        num_batches = np.ceil(len(questions)/float(batch_size) )
+        num_batches = np.ceil(len(questions)/float(batch_size))
 
         for k in range(int(num_batches)):
             batch_start = k * batch_size
@@ -234,15 +231,11 @@ class Trainer(object):
 
         tensorized_inputs = []
         for i in range(len(batch_ques)):
-            xq = Variable(self.make_input_matrix(batch_ques[i], word_vectors, vec_dim) ) #, requires_grad=False)
-            xs = Variable(self.make_input_matrix(batch_sents[i], word_vectors, vec_dim) ) #, requires_grad=False)                        
+            xq = Variable(self.make_input_matrix(batch_ques[i], word_vectors, vec_dim)) #, requires_grad=False)
+            xs = Variable(self.make_input_matrix(batch_sents[i], word_vectors, vec_dim)) #, requires_grad=False)                        
             ext_feats = Variable(torch.FloatTensor(batch_ext_feats[i]))
             ext_feats =torch.unsqueeze(ext_feats, 0)
             y[i] = batch_labels[i]
             tensorized_inputs.append((xq, xs, ext_feats))
 
         return tensorized_inputs, Variable(y)
-
-
-
-

sm-model/utils.py

@@ -1,12 +1,11 @@
 # file input output 
-# LATER model input output as well
 
 import os
 import sys
-import numpy as np
-from gensim.models.keyedvectors import KeyedVectors
 
+import numpy as np
 import torch
+from gensim.models.keyedvectors import KeyedVectors
 
 # logging setup
 import logging
@@ -19,7 +18,6 @@ formatter = logging.Formatter('%(levelname)s - %(message)s')
 ch.setFormatter(formatter)
 logger.addHandler(ch)
 
-
 def logargs(func):
     def inner(*args, **kwargs):
         logger.info('%s : %s %s' % (func.__name__, args, kwargs))
@@ -29,7 +27,7 @@ def logargs(func):
 
 def cache_word_embeddings(word_embeddings_file, cache_file):
     if not word_embeddings_file.endswith('.gz'):
-        logger.warning( 'WARNING: expecting a .gz file. Is the {} in the correct format?'.format(word_embeddings_file))
+        logger.warning('WARNING: expecting a .gz file. Is the {} in the correct format?'.format(word_embeddings_file))
 
     vocab_size, vec_dim = 0, 0
 
@@ -39,18 +37,18 @@ def cache_word_embeddings(word_embeddings_file, cache_file):
             # make cache folder if needed
             os.mkdir(os.path.dirname(cache_file))
         
-        logger.info( 'caching the word embeddings in np.memmap format'     )   
+        logger.info('caching the word embeddings in np.memmap format')   
         wv = KeyedVectors.load_word2vec_format(word_embeddings_file, binary=True)
         # print len(wv.syn0), wv.syn0.shape
         # print len(wv.syn0norm) if wv.syn0norm else None
         fp = np.memmap(cache_file, dtype=np.double, mode='w+', shape=wv.syn0.shape)
         fp[:] = wv.syn0[:]
         with open(cache_file + '.vocab', 'w') as f:
-            logger.info( 'writing out vocab for {}'.format(word_embeddings_file))
-            for _, w in sorted( (voc.index, word) for word, voc in wv.vocab.items()):
+            logger.info('writing out vocab for {}'.format(word_embeddings_file))
+            for _, w in sorted((voc.index, word) for word, voc in wv.vocab.items()):
                 print(w.encode('utf-8'), file=f)
         with open(cache_file + '.dimensions', 'w') as f:
-            logger.info( 'writing out dimensions for {}'.format(word_embeddings_file))
+            logger.info('writing out dimensions for {}'.format(word_embeddings_file))
             print(wv.syn0.shape[0], wv.syn0.shape[1], file=f)
         vocab_size, vec_dim  =  wv.syn0.shape
         del fp, wv
@@ -67,7 +65,7 @@ def load_embedding_dimensions(cache_file):
 
     
 def load_cached_embeddings(cache_file, vocab_list, oov_vec = []):
-    logger.debug( 'loading cached embeddings ')    
+    logger.debug('loading cached embeddings ')    
 
     with open(cache_file + '.dimensions') as d:
         vocab_size, vec_dim = [int(e) for e in d.read().strip().split()]
@@ -75,7 +73,7 @@ def load_cached_embeddings(cache_file, vocab_list, oov_vec = []):
     W = np.memmap(cache_file, dtype=np.double, shape=(vocab_size, vec_dim))
 
     with open(cache_file + '.vocab') as f:
-        logger.debug( 'loading vocab')
+        logger.debug('loading vocab')
         w2v_vocab_list = map(str.strip, f.readlines())
 
     vocab_dict = {w:k for k,w in enumerate(w2v_vocab_list)}
@@ -101,32 +99,33 @@ def read_in_dataset(dataset_folder, set_folder):
     max_q = 0
     max_s = 0
     set_path = os.path.join(dataset_folder, set_folder)
-    len_q_list =[ len(line.strip().split()) for line in open(os.path.join(set_path, 'a.toks')).readlines() ]
-    questions = [ line.strip() for line in open(os.path.join(set_path, 'a.toks')).readlines() ]
-    len_s_list =[ len(line.strip().split()) for line in open(os.path.join(set_path, 'b.toks')).readlines() ]
-    sentences = [ line.strip() for line in open(os.path.join(set_path, 'b.toks')).readlines() ]
-    labels = [ int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines() ]
-    ext_feats = np.array([ list(map(float, line.strip().split(' '))) for line in open(os.path.join(set_path, 'overlap_feats.txt')).readlines() ])
+    len_q_list =[len(line.strip().split()) for line in open(os.path.join(set_path, 'a.toks')).readlines()]
+    questions = [line.strip() for line in open(os.path.join(set_path, 'a.toks')).readlines()]
+    len_s_list =[len(line.strip().split()) for line in open(os.path.join(set_path, 'b.toks')).readlines()]
+    sentences = [line.strip() for line in open(os.path.join(set_path, 'b.toks')).readlines()]
+    labels = [int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines()]
+    ext_feats = np.array([list(map(float, line.strip().split(' '))) for line in open(os.path.join(set_path, 'overlap_feats.txt')).readlines()])
 
     #y = torch.from_numpy(labels)
     #return questions, sentences, y
 
-    vocab = [ line.strip() for line in open(os.path.join(dataset_folder, 'vocab.txt')).readlines() ]
+    vocab = [line.strip() for line in open(os.path.join(dataset_folder, 'vocab.txt')).readlines()]
     return questions, sentences, labels, vocab, max(len_q_list), max(len_s_list), ext_feats
 
+
 def get_test_qids_labels(dataset_folder, set_folder):
     set_path = os.path.join(dataset_folder, set_folder)
-    qids = [ line.strip() for line in open(os.path.join(set_path, 'id.txt')).readlines() ]
-    labels = np.array([ int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines() ])
+    qids = [line.strip() for line in open(os.path.join(set_path, 'id.txt')).readlines()]
+    labels = np.array([int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines()])
     return qids, labels
 
 
 if __name__ == "__main__":
     
-    vocab = [ "unk", "idontreallythinkthiswordexists", "hello" ]
+    vocab = ["unk", "idontreallythinkthiswordexists", "hello"]
 
     w2v_dict, vec_dim =  load_cached_embeddings("../../data/word2vec-models/aquaint+wiki.txt.gz.ndim=50.cache", vocab)
     
     for w, v in w2v_dict.iteritems():
         print(w)
-        print(v)
+        print(v)