Castorini smmodel (#3)

Code bridge between Castor/sm-model and Anserini

README.md

@@ -4,3 +4,22 @@ Pytorch deep learning models.
 
 1. [SM model](./sm-model/README.md): Similarity between question and candidate answers.
 
+
+## Setting up Pytorch
+
+You need Python 3.6 to use the models in this repository.
+
+As per [pytorch.org](pytorch.org) "Anaconda is our recommended package manager"
+
+```conda install pytorch torchvision -c soumith```
+
+Other pytorch installation modalities (e.g. via ```pip```) can be seen at [pytorch.org](pytorch.org).
+
+We also recommend [gensim](https://radimrehurek.com/gensim/). We use some gensim modules to cache word embeddings.
+
+```conda install gensim```
+
+
+## Castor-data
+
+Sourcing and pre-processing of input data for each model is described in respective ```model/README.md```'s

sm-model/SMCastorini.py

@@ -0,0 +1,172 @@
+import torch
+from torch.autograd import Variable
+import numpy as np
+import os
+from nltk.tokenize import TreebankWordTokenizer
+from nltk.stem.porter import *
+from collections import defaultdict
+import pickle
+import string
+
+from model import QAModel
+
+
+class SMModelCastorini(object):
+
+    def __init__(self, model_file, word_embeddings_cache_file, stopwords_file, word2dfs_file):
+        # init torch random seeds
+        torch.manual_seed(1234)
+        np.random.seed(1234)        
+
+        # load model
+        self.model = QAModel.load('', model_file)
+        # load vectors
+        self.vec_dim = self._preload_cached_embeddings(word_embeddings_cache_file)
+        self.unk_term_vec = np.random.uniform(-0.25, 0.25, self.vec_dim) 
+
+        # stopwords
+        self.stoplist = set([line.strip() for line in open(stopwords_file)])
+
+        # word dfs
+        if os.path.isfile(word2dfs_file):
+		    self.word2dfs = pickle.load( open( "word2dfs.p", "rb" ) )
+
+
+    def _preload_cached_embeddings(self, cache_file):
+        
+        with open(cache_file + '.dimensions') as d:
+            vocab_size, vec_dim = [int(e) for e in d.read().strip().split()]
+
+        self.W = np.memmap(cache_file, dtype=np.double, shape=(vocab_size, vec_dim))
+
+        with open(cache_file + '.vocab') as f:            
+            w2v_vocab_list = map(str.strip, f.readlines())
+
+        self.vocab_dict = {w:k for k,w in enumerate(w2v_vocab_list)}
+        return vec_dim
+
+
+    def parser(self, q, a):
+    	q_toks = TreebankWordTokenizer().tokenize(q)
+        q_str = ' '.join(q_toks).lower()
+        a_list = []
+        for ans in a:
+            ans_toks = TreebankWordTokenizer().tokenize(ans)
+            a_str = ' '.join(ans_toks).lower()
+            a_list.append(a_str)
+        return q_str, a_list
+
+
+    def compute_overlap_features(self, q_str, a_list, word2df=None, stoplist=None):
+    	word2df = word2df if word2df else {}
+        stoplist = stoplist if stoplist else set()
+        feats_overlap = []
+        for a in a_list:
+            question = q_str.split()
+            answer = a.split()            
+            # q_set = set(question)
+            # a_set = set(answer)
+            q_set = set([q for q in question if q not in stoplist])
+            a_set = set([a for a in answer if a not in stoplist])
+            word_overlap = q_set.intersection(a_set)
+            # overlap = float(len(word_overlap)) / (len(q_set) * len(a_set) + 1e-8)
+            if len(q_set) == 0 and len(a_set) == 0:
+                overlap = 0
+            else:
+                overlap = float(len(word_overlap)) / (len(q_set) + len(a_set))
+
+            # q_set = set([q for q in question if q not in stoplist])
+            # a_set = set([a for a in answer if a not in stoplist])
+            word_overlap = q_set.intersection(a_set)
+            df_overlap = 0.0
+            for w in word_overlap:
+                df_overlap += word2df[w]
+            
+            if len(q_set) == 0 and len(a_set) == 0:
+                df_overlap = 0
+            else:
+                df_overlap /= (len(q_set) + len(a_set))
+
+            feats_overlap.append(np.array([
+                                overlap,
+                                df_overlap,
+                                ]))
+        return np.array(feats_overlap)
+
+
+    def make_input_matrix(self,  sentence):
+        terms = sentence.strip().split()                
+        # word_embeddings = torch.zeros(max_len, vec_dim).type(torch.DoubleTensor)
+        word_embeddings = torch.zeros(len(terms), self.vec_dim).type(torch.DoubleTensor)
+        for i in range(len(terms)):
+            word = terms[i]        
+            if word not in self.vocab_dict:
+                emb = torch.from_numpy(self.unk_term_vec)
+            else:
+                emb = torch.from_numpy(self.W[self.vocab_dict[word]])                        
+            word_embeddings[i] = emb            
+
+        input_tensor = torch.zeros(1, self.vec_dim, len(terms))
+        input_tensor[0] = torch.transpose(word_embeddings, 0 , 1)
+        return input_tensor
+
+
+    def get_tensorized_inputs(self, batch_ques, batch_sents, batch_ext_feats):
+        
+        assert(1 == len(batch_ques) )
+        
+        tensorized_inputs = []
+        for i in range(len(batch_ques)):
+            xq = Variable(self.make_input_matrix(batch_ques[i]) ) 
+            xs = Variable(self.make_input_matrix(batch_sents[i]) )
+            ext_feats = Variable(torch.FloatTensor(batch_ext_feats[i]))
+            ext_feats =torch.unsqueeze(ext_feats, 0)            
+            tensorized_inputs.append((xq, xs, ext_feats))
+
+        return tensorized_inputs
+
+
+    def rerank_candidate_answers(self, question, answers):
+        # tokenize
+        q_str, a_list = self.parser(question, answers)
+
+        # calculate overlap features
+        overlap_feats = self.compute_overlap_features(q_str,a_list, stoplist=None, word2df=self.word2dfs)        
+        overlap_feats_stoplist = self.compute_overlap_features(q_str, a_list, stoplist=self.stoplist, word2df=self.word2dfs)
+        overlap_feats_vec = np.hstack([overlap_feats, overlap_feats_stoplist])
+
+        # run through the model
+        scores_sentences = []
+        for i in range(len(a_list)):
+            xq, xa, x_ext_feats = self.get_tensorized_inputs([q], [a_list[i]], [overlap_feats_vec[i]])[0]          
+            pred = self.model(xq, xa, x_ext_feats)               
+            pred = torch.exp(pred)
+            scores_sentences.append( (pred.data.squeeze()[1], a_list[i]) )
+        
+        return scores_sentences
+
+
+if __name__ == "__main__":
+    
+
+    smmodel = SMModelCastorini('../../data/TrecQA/sm.model.aquaint.castorini', 
+                    '../../data/word2vec-models/aquaint+wiki.txt.gz.ndim=50.cache',
+                    'stopwords.txt',
+                    'word2dfs.p')
+    
+    
+    q = "who is the author of the book , `` the iron lady : a biography of margaret thatcher '' ?"
+    a = [ 
+    "the iron lady ; a biography of margaret thatcher by hugo young -lrb- farrar , straus & giroux -rrb-",
+    "in this same revisionist mold , hugo young , the distinguished british journalist , has performed a brilliant dissection of the notion of thatcher as a conservative icon .",
+    "in `` the iron lady , '' young traces the winding staircase of fortune that transformed the younger daughter of a provincial english grocer into the greatest woman political leader since catherine the great .",
+    "`` he is the very essence of the classless meritocrat , '' says hugo young , thatcher 's biographer .",
+    "from her father , young argues , she inherited a `` joyless earnestness '' that combined with her early interest in science to produce the roots of her public character .",
+    "this is not the answer",
+    "asdfawe asdf sertse dgfsgsfg"
+    ]
+	
+    ss = smmodel.rerank_candidate_answers(q, a)
+    print('Question:', q)
+    for score, sentence in ss:
+        print(score, '\t', sentence)

sm-model/overlap_features.py

@@ -12,7 +12,7 @@ from collections import defaultdict
 def load_data(dname):
   stemmer = PorterStemmer()
   qids, questions, answers, labels = [], [], [], []
-  print dname
+  print(dname)
   with open(dname+'a.toks') as f:
     for line in f:
       line = unicode(line, errors='ignore')
@@ -148,33 +148,33 @@ if __name__ == '__main__':
   
   docs = all_answers + unique_questions
   word2dfs = compute_dfs(docs)
-  print word2dfs.items()[:10]
+  print(word2dfs.items()[:10])
 
   q_max_sent_length = max(map(lambda x: len(x), all_questions))
   a_max_sent_length = max(map(lambda x: len(x), all_answers))
-  print 'q_max_sent_length', q_max_sent_length
-  print 'a_max_sent_length', a_max_sent_length
+  print('q_max_sent_length', q_max_sent_length)
+  print('a_max_sent_length', a_max_sent_length)
 
   for sub in sub_dirs:
-    print sub
+    print(sub)
     qids, questions, answers, labels = load_data(base_dir+sub)
 
     overlap_feats = compute_overlap_features(questions, answers, stoplist=None, word2df=word2dfs)
     overlap_feats_stoplist = compute_overlap_features(questions, answers, stoplist=stoplist, word2df=word2dfs)
     overlap_feats = np.hstack([overlap_feats, overlap_feats_stoplist])
-    print overlap_feats[:3]
-    print 'overlap_feats', overlap_feats.shape
+    print(overlap_feats[:3])
+    print('overlap_feats', overlap_feats.shape)
     
     from sklearn.preprocessing import StandardScaler
     scaler = StandardScaler()
-    print "Scaling overlap features"
+    print("Scaling overlap features")
     overlap_feats = scaler.fit_transform(overlap_feats)
-    print overlap_feats[:3]    
+    print(overlap_feats[:3]    )
 
     '''q_overlap_indices, a_overlap_indices = compute_overlap_idx(questions, answers, stoplist, q_max_sent_length, a_max_sent_length)
-    print q_overlap_indices[:3], a_overlap_indices[:3]
-    print 'q_overlap_indices', q_overlap_indices.shape
-    print 'a_overlap_indices', a_overlap_indices.shape'''
+    print(q_overlap_indices[:3], a_overlap_indices[:3])
+    print('q_overlap_indices', q_overlap_indices.shape)
+    print('a_overlap_indices', a_overlap_indices.shape)'''
 
     with open(base_dir+sub+'overlap_feats.txt', 'w') as f:
       for i in range(overlap_feats.shape[0]):

sm-model/train.py

@@ -127,7 +127,7 @@ class Trainer(object):
         y_pred = np.zeros(len(questions))
         ypc = 0
             
-        for k in xrange(int(num_batches)):
+        for k in range(int(num_batches)):
             batch_start = k * batch_size
             batch_end = (k+1) * batch_size
             # convert raw questions and sentences to tensors
@@ -165,14 +165,14 @@ class Trainer(object):
 
         questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = self.datasets[set_folder]
         word_vectors, vec_dim = self.embeddings[set_folder], self.vec_dim
-
+        
         # set model for training modep
         self.model.train()
 
         train_loss, train_correct = 0., 0.
         num_batches = np.ceil(len(questions)/float(batch_size) )
 
-        for k in xrange(int(num_batches)):
+        for k in range(int(num_batches)):
             batch_start = k * batch_size
             batch_end = (k+1) * batch_size
 
@@ -210,7 +210,7 @@ class Trainer(object):
         terms = sentence.strip().split()        
         # word_embeddings = torch.zeros(max_len, vec_dim).type(torch.DoubleTensor)
         word_embeddings = torch.zeros(len(terms), vec_dim).type(torch.DoubleTensor)
-        for i in xrange(len(terms)):
+        for i in range(len(terms)):
             word = terms[i]        
             emb = torch.from_numpy(word_vectors[word])                        
             word_embeddings[i] = emb            
@@ -233,9 +233,9 @@ class Trainer(object):
         y = torch.LongTensor(batch_size).type(torch.LongTensor)
 
         tensorized_inputs = []
-        for i in xrange(len(batch_ques)):
+        for i in range(len(batch_ques)):
             xq = Variable(self.make_input_matrix(batch_ques[i], word_vectors, vec_dim) ) #, requires_grad=False)
-            xs = Variable(self.make_input_matrix(batch_sents[i], word_vectors, vec_dim) ) #, requires_grad=False)            
+            xs = Variable(self.make_input_matrix(batch_sents[i], word_vectors, vec_dim) ) #, requires_grad=False)                        
             ext_feats = Variable(torch.FloatTensor(batch_ext_feats[i]))
             ext_feats =torch.unsqueeze(ext_feats, 0)
             y[i] = batch_labels[i]

sm-model/utils.py

@@ -8,10 +8,6 @@ from gensim.models.keyedvectors import KeyedVectors
 
 import torch
 
-import cPickle
-
-
-
 # logging setup
 import logging
 logger = logging.getLogger(__name__)
@@ -24,41 +20,6 @@ ch.setFormatter(formatter)
 logger.addHandler(ch)
 
 
-
-def load_bin_vec(fname, words):
-  """
-  Loads 300x1 word vecs from Google (Mikolov) word2vec
-  """
-  print fname
-  vocab = set(words)
-  word_vecs = {}
-  with open(fname, "rb") as f:
-    header = f.readline()
-    vocab_size, layer1_size = map(int, header.split())
-    binary_len = numpy.dtype('float32').itemsize * layer1_size
-    print 'vocab_size, layer1_size', vocab_size, layer1_size
-    count = 0
-    for i, line in enumerate(xrange(vocab_size)):
-      if i % 100000 == 0:
-        print '.',
-      word = []
-      while True:
-        ch = f.read(1)
-        if ch == ' ':
-            word = ''.join(word)
-            break
-        if ch != '\n':
-            word.append(ch)
-      if word in vocab:
-        count += 1
-        word_vecs[word] = numpy.fromstring(f.read(binary_len), dtype='float32')
-      else:
-          f.read(binary_len)
-    print "done"
-    print "Words found in wor2vec embeddings", count
-    return word_vecs
-
-
 def logargs(func):
     def inner(*args, **kwargs):
         logger.info('%s : %s %s' % (func.__name__, args, kwargs))
@@ -87,13 +48,13 @@ def cache_word_embeddings(word_embeddings_file, cache_file):
         with open(cache_file + '.vocab', 'w') as f:
             logger.info( 'writing out vocab for {}'.format(word_embeddings_file))
             for _, w in sorted( (voc.index, word) for word, voc in wv.vocab.items()):
-                print >> f, w.encode('utf-8')
+                print(w.encode('utf-8'), file=f)
         with open(cache_file + '.dimensions', 'w') as f:
             logger.info( 'writing out dimensions for {}'.format(word_embeddings_file))
-            print >> f, wv.syn0.shape[0], wv.syn0.shape[1]
+            print(wv.syn0.shape[0], wv.syn0.shape[1], file=f)
         vocab_size, vec_dim  =  wv.syn0.shape
         del fp, wv
-        print 'cached {} into {}'.format(word_embeddings_file, cache_file)
+        print('cached {} into {}'.format(word_embeddings_file, cache_file))
 
     return vocab_size, vec_dim
 
@@ -144,8 +105,8 @@ def read_in_dataset(dataset_folder, set_folder):
     questions = [ line.strip() for line in open(os.path.join(set_path, 'a.toks')).readlines() ]
     len_s_list =[ len(line.strip().split()) for line in open(os.path.join(set_path, 'b.toks')).readlines() ]
     sentences = [ line.strip() for line in open(os.path.join(set_path, 'b.toks')).readlines() ]
-    labels = np.array([ int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines() ])
-    ext_feats = np.array([ map(float, line.strip().split(' ')) for line in open(os.path.join(set_path, 'overlap_feats.txt')).readlines() ])
+    labels = [ int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines() ]
+    ext_feats = np.array([ list(map(float, line.strip().split(' '))) for line in open(os.path.join(set_path, 'overlap_feats.txt')).readlines() ])
 
     #y = torch.from_numpy(labels)
     #return questions, sentences, y
@@ -167,5 +128,5 @@ if __name__ == "__main__":
     w2v_dict, vec_dim =  load_cached_embeddings("../../data/word2vec-models/aquaint+wiki.txt.gz.ndim=50.cache", vocab)
     
     for w, v in w2v_dict.iteritems():
-        print w
-        print v
+        print(w)
+        print(v)