From 73577acc3f7dc63571dd1f76980b2e6082a0ac3a Mon Sep 17 00:00:00 2001 From: gauravbaruah Date: Fri, 31 Mar 2017 15:30:57 -0400 Subject: [PATCH] Castorini smmodel (#3) Code bridge between Castor/sm-model and Anserini --- README.md | 19 ++++ sm-model/SMCastorini.py | 172 +++++++++++++++++++++++++++++++++++ sm-model/overlap_features.py | 24 ++--- sm-model/train.py | 12 +-- sm-model/utils.py | 53 ++--------- 5 files changed, 216 insertions(+), 64 deletions(-) create mode 100644 sm-model/SMCastorini.py diff --git a/README.md b/README.md index ae884ad..ef2eaa2 100644 --- a/README.md +++ b/README.md @@ -4,3 +4,22 @@ Pytorch deep learning models. 1. [SM model](./sm-model/README.md): Similarity between question and candidate answers. + +## Setting up Pytorch + +You need Python 3.6 to use the models in this repository. + +As per [pytorch.org](pytorch.org) "Anaconda is our recommended package manager" + +```conda install pytorch torchvision -c soumith``` + +Other pytorch installation modalities (e.g. via ```pip```) can be seen at [pytorch.org](pytorch.org). + +We also recommend [gensim](https://radimrehurek.com/gensim/). We use some gensim modules to cache word embeddings. + +```conda install gensim``` + + +## Castor-data + +Sourcing and pre-processing of input data for each model is described in respective ```model/README.md```'s \ No newline at end of file diff --git a/sm-model/SMCastorini.py b/sm-model/SMCastorini.py new file mode 100644 index 0000000..3a7ed74 --- /dev/null +++ b/sm-model/SMCastorini.py @@ -0,0 +1,172 @@ +import torch +from torch.autograd import Variable +import numpy as np +import os +from nltk.tokenize import TreebankWordTokenizer +from nltk.stem.porter import * +from collections import defaultdict +import pickle +import string + +from model import QAModel + + +class SMModelCastorini(object): + + def __init__(self, model_file, word_embeddings_cache_file, stopwords_file, word2dfs_file): + # init torch random seeds + torch.manual_seed(1234) + np.random.seed(1234) + + # load model + self.model = QAModel.load('', model_file) + # load vectors + self.vec_dim = self._preload_cached_embeddings(word_embeddings_cache_file) + self.unk_term_vec = np.random.uniform(-0.25, 0.25, self.vec_dim) + + # stopwords + self.stoplist = set([line.strip() for line in open(stopwords_file)]) + + # word dfs + if os.path.isfile(word2dfs_file): + self.word2dfs = pickle.load( open( "word2dfs.p", "rb" ) ) + + + def _preload_cached_embeddings(self, cache_file): + + with open(cache_file + '.dimensions') as d: + vocab_size, vec_dim = [int(e) for e in d.read().strip().split()] + + self.W = np.memmap(cache_file, dtype=np.double, shape=(vocab_size, vec_dim)) + + with open(cache_file + '.vocab') as f: + w2v_vocab_list = map(str.strip, f.readlines()) + + self.vocab_dict = {w:k for k,w in enumerate(w2v_vocab_list)} + return vec_dim + + + def parser(self, q, a): + q_toks = TreebankWordTokenizer().tokenize(q) + q_str = ' '.join(q_toks).lower() + a_list = [] + for ans in a: + ans_toks = TreebankWordTokenizer().tokenize(ans) + a_str = ' '.join(ans_toks).lower() + a_list.append(a_str) + return q_str, a_list + + + def compute_overlap_features(self, q_str, a_list, word2df=None, stoplist=None): + word2df = word2df if word2df else {} + stoplist = stoplist if stoplist else set() + feats_overlap = [] + for a in a_list: + question = q_str.split() + answer = a.split() + # q_set = set(question) + # a_set = set(answer) + q_set = set([q for q in question if q not in stoplist]) + a_set = set([a for a in answer if a not in stoplist]) + word_overlap = q_set.intersection(a_set) + # overlap = float(len(word_overlap)) / (len(q_set) * len(a_set) + 1e-8) + if len(q_set) == 0 and len(a_set) == 0: + overlap = 0 + else: + overlap = float(len(word_overlap)) / (len(q_set) + len(a_set)) + + # q_set = set([q for q in question if q not in stoplist]) + # a_set = set([a for a in answer if a not in stoplist]) + word_overlap = q_set.intersection(a_set) + df_overlap = 0.0 + for w in word_overlap: + df_overlap += word2df[w] + + if len(q_set) == 0 and len(a_set) == 0: + df_overlap = 0 + else: + df_overlap /= (len(q_set) + len(a_set)) + + feats_overlap.append(np.array([ + overlap, + df_overlap, + ])) + return np.array(feats_overlap) + + + def make_input_matrix(self, sentence): + terms = sentence.strip().split() + # word_embeddings = torch.zeros(max_len, vec_dim).type(torch.DoubleTensor) + word_embeddings = torch.zeros(len(terms), self.vec_dim).type(torch.DoubleTensor) + for i in range(len(terms)): + word = terms[i] + if word not in self.vocab_dict: + emb = torch.from_numpy(self.unk_term_vec) + else: + emb = torch.from_numpy(self.W[self.vocab_dict[word]]) + word_embeddings[i] = emb + + input_tensor = torch.zeros(1, self.vec_dim, len(terms)) + input_tensor[0] = torch.transpose(word_embeddings, 0 , 1) + return input_tensor + + + def get_tensorized_inputs(self, batch_ques, batch_sents, batch_ext_feats): + + assert(1 == len(batch_ques) ) + + tensorized_inputs = [] + for i in range(len(batch_ques)): + xq = Variable(self.make_input_matrix(batch_ques[i]) ) + xs = Variable(self.make_input_matrix(batch_sents[i]) ) + ext_feats = Variable(torch.FloatTensor(batch_ext_feats[i])) + ext_feats =torch.unsqueeze(ext_feats, 0) + tensorized_inputs.append((xq, xs, ext_feats)) + + return tensorized_inputs + + + def rerank_candidate_answers(self, question, answers): + # tokenize + q_str, a_list = self.parser(question, answers) + + # calculate overlap features + overlap_feats = self.compute_overlap_features(q_str,a_list, stoplist=None, word2df=self.word2dfs) + overlap_feats_stoplist = self.compute_overlap_features(q_str, a_list, stoplist=self.stoplist, word2df=self.word2dfs) + overlap_feats_vec = np.hstack([overlap_feats, overlap_feats_stoplist]) + + # run through the model + scores_sentences = [] + for i in range(len(a_list)): + xq, xa, x_ext_feats = self.get_tensorized_inputs([q], [a_list[i]], [overlap_feats_vec[i]])[0] + pred = self.model(xq, xa, x_ext_feats) + pred = torch.exp(pred) + scores_sentences.append( (pred.data.squeeze()[1], a_list[i]) ) + + return scores_sentences + + +if __name__ == "__main__": + + + smmodel = SMModelCastorini('../../data/TrecQA/sm.model.aquaint.castorini', + '../../data/word2vec-models/aquaint+wiki.txt.gz.ndim=50.cache', + 'stopwords.txt', + 'word2dfs.p') + + + q = "who is the author of the book , `` the iron lady : a biography of margaret thatcher '' ?" + a = [ + "the iron lady ; a biography of margaret thatcher by hugo young -lrb- farrar , straus & giroux -rrb-", + "in this same revisionist mold , hugo young , the distinguished british journalist , has performed a brilliant dissection of the notion of thatcher as a conservative icon .", + "in `` the iron lady , '' young traces the winding staircase of fortune that transformed the younger daughter of a provincial english grocer into the greatest woman political leader since catherine the great .", + "`` he is the very essence of the classless meritocrat , '' says hugo young , thatcher 's biographer .", + "from her father , young argues , she inherited a `` joyless earnestness '' that combined with her early interest in science to produce the roots of her public character .", + "this is not the answer", + "asdfawe asdf sertse dgfsgsfg" + ] + + ss = smmodel.rerank_candidate_answers(q, a) + print('Question:', q) + for score, sentence in ss: + print(score, '\t', sentence) \ No newline at end of file diff --git a/sm-model/overlap_features.py b/sm-model/overlap_features.py index 97b677f..cdb0bc2 100644 --- a/sm-model/overlap_features.py +++ b/sm-model/overlap_features.py @@ -12,7 +12,7 @@ from collections import defaultdict def load_data(dname): stemmer = PorterStemmer() qids, questions, answers, labels = [], [], [], [] - print dname + print(dname) with open(dname+'a.toks') as f: for line in f: line = unicode(line, errors='ignore') @@ -148,33 +148,33 @@ if __name__ == '__main__': docs = all_answers + unique_questions word2dfs = compute_dfs(docs) - print word2dfs.items()[:10] + print(word2dfs.items()[:10]) q_max_sent_length = max(map(lambda x: len(x), all_questions)) a_max_sent_length = max(map(lambda x: len(x), all_answers)) - print 'q_max_sent_length', q_max_sent_length - print 'a_max_sent_length', a_max_sent_length + print('q_max_sent_length', q_max_sent_length) + print('a_max_sent_length', a_max_sent_length) for sub in sub_dirs: - print sub + print(sub) qids, questions, answers, labels = load_data(base_dir+sub) overlap_feats = compute_overlap_features(questions, answers, stoplist=None, word2df=word2dfs) overlap_feats_stoplist = compute_overlap_features(questions, answers, stoplist=stoplist, word2df=word2dfs) overlap_feats = np.hstack([overlap_feats, overlap_feats_stoplist]) - print overlap_feats[:3] - print 'overlap_feats', overlap_feats.shape + print(overlap_feats[:3]) + print('overlap_feats', overlap_feats.shape) from sklearn.preprocessing import StandardScaler scaler = StandardScaler() - print "Scaling overlap features" + print("Scaling overlap features") overlap_feats = scaler.fit_transform(overlap_feats) - print overlap_feats[:3] + print(overlap_feats[:3] ) '''q_overlap_indices, a_overlap_indices = compute_overlap_idx(questions, answers, stoplist, q_max_sent_length, a_max_sent_length) - print q_overlap_indices[:3], a_overlap_indices[:3] - print 'q_overlap_indices', q_overlap_indices.shape - print 'a_overlap_indices', a_overlap_indices.shape''' + print(q_overlap_indices[:3], a_overlap_indices[:3]) + print('q_overlap_indices', q_overlap_indices.shape) + print('a_overlap_indices', a_overlap_indices.shape)''' with open(base_dir+sub+'overlap_feats.txt', 'w') as f: for i in range(overlap_feats.shape[0]): diff --git a/sm-model/train.py b/sm-model/train.py index 19214d3..1f969f1 100644 --- a/sm-model/train.py +++ b/sm-model/train.py @@ -127,7 +127,7 @@ class Trainer(object): y_pred = np.zeros(len(questions)) ypc = 0 - for k in xrange(int(num_batches)): + for k in range(int(num_batches)): batch_start = k * batch_size batch_end = (k+1) * batch_size # convert raw questions and sentences to tensors @@ -165,14 +165,14 @@ class Trainer(object): questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = self.datasets[set_folder] word_vectors, vec_dim = self.embeddings[set_folder], self.vec_dim - + # set model for training modep self.model.train() train_loss, train_correct = 0., 0. num_batches = np.ceil(len(questions)/float(batch_size) ) - for k in xrange(int(num_batches)): + for k in range(int(num_batches)): batch_start = k * batch_size batch_end = (k+1) * batch_size @@ -210,7 +210,7 @@ class Trainer(object): terms = sentence.strip().split() # word_embeddings = torch.zeros(max_len, vec_dim).type(torch.DoubleTensor) word_embeddings = torch.zeros(len(terms), vec_dim).type(torch.DoubleTensor) - for i in xrange(len(terms)): + for i in range(len(terms)): word = terms[i] emb = torch.from_numpy(word_vectors[word]) word_embeddings[i] = emb @@ -233,9 +233,9 @@ class Trainer(object): y = torch.LongTensor(batch_size).type(torch.LongTensor) tensorized_inputs = [] - for i in xrange(len(batch_ques)): + for i in range(len(batch_ques)): xq = Variable(self.make_input_matrix(batch_ques[i], word_vectors, vec_dim) ) #, requires_grad=False) - xs = Variable(self.make_input_matrix(batch_sents[i], word_vectors, vec_dim) ) #, requires_grad=False) + xs = Variable(self.make_input_matrix(batch_sents[i], word_vectors, vec_dim) ) #, requires_grad=False) ext_feats = Variable(torch.FloatTensor(batch_ext_feats[i])) ext_feats =torch.unsqueeze(ext_feats, 0) y[i] = batch_labels[i] diff --git a/sm-model/utils.py b/sm-model/utils.py index 44e6056..572b3c7 100644 --- a/sm-model/utils.py +++ b/sm-model/utils.py @@ -8,10 +8,6 @@ from gensim.models.keyedvectors import KeyedVectors import torch -import cPickle - - - # logging setup import logging logger = logging.getLogger(__name__) @@ -24,41 +20,6 @@ ch.setFormatter(formatter) logger.addHandler(ch) - -def load_bin_vec(fname, words): - """ - Loads 300x1 word vecs from Google (Mikolov) word2vec - """ - print fname - vocab = set(words) - word_vecs = {} - with open(fname, "rb") as f: - header = f.readline() - vocab_size, layer1_size = map(int, header.split()) - binary_len = numpy.dtype('float32').itemsize * layer1_size - print 'vocab_size, layer1_size', vocab_size, layer1_size - count = 0 - for i, line in enumerate(xrange(vocab_size)): - if i % 100000 == 0: - print '.', - word = [] - while True: - ch = f.read(1) - if ch == ' ': - word = ''.join(word) - break - if ch != '\n': - word.append(ch) - if word in vocab: - count += 1 - word_vecs[word] = numpy.fromstring(f.read(binary_len), dtype='float32') - else: - f.read(binary_len) - print "done" - print "Words found in wor2vec embeddings", count - return word_vecs - - def logargs(func): def inner(*args, **kwargs): logger.info('%s : %s %s' % (func.__name__, args, kwargs)) @@ -87,13 +48,13 @@ def cache_word_embeddings(word_embeddings_file, cache_file): with open(cache_file + '.vocab', 'w') as f: logger.info( 'writing out vocab for {}'.format(word_embeddings_file)) for _, w in sorted( (voc.index, word) for word, voc in wv.vocab.items()): - print >> f, w.encode('utf-8') + print(w.encode('utf-8'), file=f) with open(cache_file + '.dimensions', 'w') as f: logger.info( 'writing out dimensions for {}'.format(word_embeddings_file)) - print >> f, wv.syn0.shape[0], wv.syn0.shape[1] + print(wv.syn0.shape[0], wv.syn0.shape[1], file=f) vocab_size, vec_dim = wv.syn0.shape del fp, wv - print 'cached {} into {}'.format(word_embeddings_file, cache_file) + print('cached {} into {}'.format(word_embeddings_file, cache_file)) return vocab_size, vec_dim @@ -144,8 +105,8 @@ def read_in_dataset(dataset_folder, set_folder): questions = [ line.strip() for line in open(os.path.join(set_path, 'a.toks')).readlines() ] len_s_list =[ len(line.strip().split()) for line in open(os.path.join(set_path, 'b.toks')).readlines() ] sentences = [ line.strip() for line in open(os.path.join(set_path, 'b.toks')).readlines() ] - labels = np.array([ int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines() ]) - ext_feats = np.array([ map(float, line.strip().split(' ')) for line in open(os.path.join(set_path, 'overlap_feats.txt')).readlines() ]) + labels = [ int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines() ] + ext_feats = np.array([ list(map(float, line.strip().split(' '))) for line in open(os.path.join(set_path, 'overlap_feats.txt')).readlines() ]) #y = torch.from_numpy(labels) #return questions, sentences, y @@ -167,5 +128,5 @@ if __name__ == "__main__": w2v_dict, vec_dim = load_cached_embeddings("../../data/word2vec-models/aquaint+wiki.txt.gz.ndim=50.cache", vocab) for w, v in w2v_dict.iteritems(): - print w - print v \ No newline at end of file + print(w) + print(v) \ No newline at end of file