diff --git a/README.md b/README.md index ef2eaa2..b71b904 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,8 @@ Pytorch deep learning models. You need Python 3.6 to use the models in this repository. -As per [pytorch.org](pytorch.org) "Anaconda is our recommended package manager" +As per [pytorch.org](pytorch.org) +> [Anaconda](https://www.continuum.io/downloads) is our recommended package manager ```conda install pytorch torchvision -c soumith``` @@ -20,6 +21,12 @@ We also recommend [gensim](https://radimrehurek.com/gensim/). We use some gensim ```conda install gensim``` +Pytorch has good support for GPU computations. +CUDA installation guide for linux can be found [here](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/) + +**NOTE**: Install CUDA libraries **before** installing conda and pytorch. + + ## Castor-data Sourcing and pre-processing of input data for each model is described in respective ```model/README.md```'s \ No newline at end of file diff --git a/sm-model/README.md b/sm-model/README.md index 9e1ba8b..2455be0 100644 --- a/sm-model/README.md +++ b/sm-model/README.md @@ -8,6 +8,9 @@ 1. figure out if the L2 regularization is correct 2. Batch size of 50 (current batch_size = 1) +#### Getting the data + +TODO: #### Running it diff --git a/sm-model/SMCastorini.py b/sm-model/SMCastorini.py index 3a7ed74..5858a24 100644 --- a/sm-model/SMCastorini.py +++ b/sm-model/SMCastorini.py @@ -1,12 +1,12 @@ -import torch -from torch.autograd import Variable -import numpy as np import os -from nltk.tokenize import TreebankWordTokenizer -from nltk.stem.porter import * -from collections import defaultdict import pickle import string +from collections import defaultdict + +import numpy as np +import torch +from nltk.tokenize import TreebankWordTokenizer +from torch.autograd import Variable from model import QAModel @@ -29,7 +29,8 @@ class SMModelCastorini(object): # word dfs if os.path.isfile(word2dfs_file): - self.word2dfs = pickle.load( open( "word2dfs.p", "rb" ) ) + with open(word2dfs_file, "rb") as w2dfin: + self.word2dfs = pickle.load(w2dfin) def _preload_cached_embeddings(self, cache_file): @@ -47,7 +48,7 @@ class SMModelCastorini(object): def parser(self, q, a): - q_toks = TreebankWordTokenizer().tokenize(q) + q_toks = TreebankWordTokenizer().tokenize(q) q_str = ' '.join(q_toks).lower() a_list = [] for ans in a: @@ -58,7 +59,7 @@ class SMModelCastorini(object): def compute_overlap_features(self, q_str, a_list, word2df=None, stoplist=None): - word2df = word2df if word2df else {} + word2df = word2df if word2df else {} stoplist = stoplist if stoplist else set() feats_overlap = [] for a in a_list: @@ -105,24 +106,20 @@ class SMModelCastorini(object): else: emb = torch.from_numpy(self.W[self.vocab_dict[word]]) word_embeddings[i] = emb - input_tensor = torch.zeros(1, self.vec_dim, len(terms)) - input_tensor[0] = torch.transpose(word_embeddings, 0 , 1) + input_tensor[0] = torch.transpose(word_embeddings, 0, 1) return input_tensor - def get_tensorized_inputs(self, batch_ques, batch_sents, batch_ext_feats): - - assert(1 == len(batch_ques) ) - + def get_tensorized_inputs(self, batch_ques, batch_sents, batch_ext_feats): + assert(1 == len(batch_ques)) tensorized_inputs = [] for i in range(len(batch_ques)): - xq = Variable(self.make_input_matrix(batch_ques[i]) ) - xs = Variable(self.make_input_matrix(batch_sents[i]) ) + xq = Variable(self.make_input_matrix(batch_ques[i])) + xs = Variable(self.make_input_matrix(batch_sents[i])) ext_feats = Variable(torch.FloatTensor(batch_ext_feats[i])) ext_feats =torch.unsqueeze(ext_feats, 0) tensorized_inputs.append((xq, xs, ext_feats)) - return tensorized_inputs @@ -141,32 +138,33 @@ class SMModelCastorini(object): xq, xa, x_ext_feats = self.get_tensorized_inputs([q], [a_list[i]], [overlap_feats_vec[i]])[0] pred = self.model(xq, xa, x_ext_feats) pred = torch.exp(pred) - scores_sentences.append( (pred.data.squeeze()[1], a_list[i]) ) + scores_sentences.append((pred.data.squeeze()[1], a_list[i])) return scores_sentences if __name__ == "__main__": - - smmodel = SMModelCastorini('../../data/TrecQA/sm.model.aquaint.castorini', + smmodel = SMModelCastorini('sm.model.py3', '../../data/word2vec-models/aquaint+wiki.txt.gz.ndim=50.cache', 'stopwords.txt', 'word2dfs.p') - - + q = "who is the author of the book , `` the iron lady : a biography of margaret thatcher '' ?" a = [ - "the iron lady ; a biography of margaret thatcher by hugo young -lrb- farrar , straus & giroux -rrb-", - "in this same revisionist mold , hugo young , the distinguished british journalist , has performed a brilliant dissection of the notion of thatcher as a conservative icon .", - "in `` the iron lady , '' young traces the winding staircase of fortune that transformed the younger daughter of a provincial english grocer into the greatest woman political leader since catherine the great .", - "`` he is the very essence of the classless meritocrat , '' says hugo young , thatcher 's biographer .", - "from her father , young argues , she inherited a `` joyless earnestness '' that combined with her early interest in science to produce the roots of her public character .", - "this is not the answer", - "asdfawe asdf sertse dgfsgsfg" + "the iron lady ; a biography of margaret thatcher by hugo young -lrb- farrar , straus & giroux -rrb-", + "in this same revisionist mold , hugo young , the distinguished british journalist , has performed a brilliant \ + dissection of the notion of thatcher as a conservative icon .", + "in `` the iron lady , '' young traces the winding staircase of fortune that transformed the younger daughter \ + of a provincial english grocer into the greatest woman political leader since catherine the great .", + "`` he is the very essence of the classless meritocrat , '' says hugo young , thatcher 's biographer .", + "from her father , young argues , she inherited a `` joyless earnestness '' that combined with her early \ + interest in science to produce the roots of her public character .", + "this is not the answer", + "asdfawe asdf sertse dgfsgsfg" ] ss = smmodel.rerank_candidate_answers(q, a) print('Question:', q) for score, sentence in ss: - print(score, '\t', sentence) \ No newline at end of file + print(score, '\t', sentence) diff --git a/sm-model/main.py b/sm-model/main.py index 52e69e2..d0f8b3b 100644 --- a/sm-model/main.py +++ b/sm-model/main.py @@ -1,22 +1,19 @@ -import os -import sys -import time -import glob import argparse -import numpy as np -import pandas as pd -import subprocess +import os import shlex +import subprocess +import sys +import numpy as np +import pandas as pd import torch -import torch.optim as optim import torch.nn as nn +import torch.optim as optim from torch.autograd import Variable - -from model import QAModel import utils +from model import QAModel from train import Trainer # logging setup @@ -38,7 +35,7 @@ def logargs(func): def compute_map_mrr(dataset_folder, set_folder, test_scores): - # logger.info( "Running trec_eval script..." ) + # logger.info("Running trec_eval script...") N = len(test_scores) qids_test, y_test = utils.get_test_qids_labels(dataset_folder, set_folder) @@ -65,7 +62,7 @@ def compute_map_mrr(dataset_folder, set_folder, test_scores): p = subprocess.Popen(pargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE) pout, perr = p.communicate() - lines = pout.split('\n') + lines = pout.split(b'\n') map = float(lines[0].strip().split()[-1]) mrr = float(lines[1].strip().split()[-1]) return map, mrr @@ -165,5 +162,3 @@ if __name__ == "__main__": map, mrr = compute_map_mrr(args.dataset_folder, test_set, test_scores) logger.info("------- MAP {}, MRR {}".format(map, mrr)) - - diff --git a/sm-model/model.py b/sm-model/model.py index 72fcfcf..22f1207 100644 --- a/sm-model/model.py +++ b/sm-model/model.py @@ -1,10 +1,10 @@ +import os + +import numpy as np import torch import torch.nn as nn import torch.nn.functional as F from torch.autograd import Variable -import numpy as np - -import os # logging setup import logging @@ -23,11 +23,13 @@ class QAModel(nn.Module): @staticmethod def save(model, out_folder, model_fname): torch.save(model, os.path.join(out_folder, model_fname)) + @staticmethod def load(in_folder, model_fname): return torch.load(os.path.join(in_folder, model_fname)) - + + def __init__(self, input_n_dim, filter_width, conv_filters=100, no_ext_feats=False, ext_feats_size=4, n_classes=2): super(QAModel, self).__init__() diff --git a/sm-model/overlap_features.py b/sm-model/overlap_features.py index cdb0bc2..3dd68bf 100644 --- a/sm-model/overlap_features.py +++ b/sm-model/overlap_features.py @@ -169,7 +169,7 @@ if __name__ == '__main__': scaler = StandardScaler() print("Scaling overlap features") overlap_feats = scaler.fit_transform(overlap_feats) - print(overlap_feats[:3] ) + print(overlap_feats[:3]) '''q_overlap_indices, a_overlap_indices = compute_overlap_idx(questions, answers, stoplist, q_max_sent_length, a_max_sent_length) print(q_overlap_indices[:3], a_overlap_indices[:3]) diff --git a/sm-model/train.py b/sm-model/train.py index 1f969f1..9579850 100644 --- a/sm-model/train.py +++ b/sm-model/train.py @@ -1,14 +1,11 @@ -import os -import sys +import argparse import time -import glob -import argparse -import numpy as np +import numpy as np import torch -import torch.optim as optim import torch.nn as nn +import torch.optim as optim from torch.autograd import Variable import utils @@ -39,7 +36,7 @@ class Trainer(object): self.model = model self.criterion = nn.CrossEntropyLoss() #self.criterion = nn.NLLLoss() - self.optimizer = optim.SGD(self.model.parameters(), lr=eta, momentum=mom, weight_decay=(0 if no_loss_reg else self.reg) ) + self.optimizer = optim.SGD(self.model.parameters(), lr=eta, momentum=mom, weight_decay=(0 if no_loss_reg else self.reg)) self.datasets = {} self.embeddings = {} @@ -123,7 +120,7 @@ class Trainer(object): total_loss = 0.0 total_correct = 0.0 - num_batches = np.ceil(len(questions)/batch_size ) + num_batches = np.ceil(len(questions)/batch_size) y_pred = np.zeros(len(questions)) ypc = 0 @@ -154,8 +151,8 @@ class Trainer(object): # logger.info('{}_correct {}'.format(set_folder, total_correct)) # logger.info('{}_loss {}'.format(set_folder, total_loss.data[0])) logger.info('{} total {}'.format(set_folder, len(labels))) - # logger.info('{}_loss = {:.4f}, acc = {:.4f}'.format( set_folder, total_loss.data[0]/len(labels), float(total_correct)/len(labels) )) - #logger.info('{}_loss = {:.4f}'.format( set_folder, total_loss.data[0]/len(labels) )) + # logger.info('{}_loss = {:.4f}, acc = {:.4f}'.format(set_folder, total_loss.data[0]/len(labels), float(total_correct)/len(labels)) + #logger.info('{}_loss = {:.4f}'.format(set_folder, total_loss.data[0]/len(labels))) return y_pred @@ -170,7 +167,7 @@ class Trainer(object): self.model.train() train_loss, train_correct = 0., 0. - num_batches = np.ceil(len(questions)/float(batch_size) ) + num_batches = np.ceil(len(questions)/float(batch_size)) for k in range(int(num_batches)): batch_start = k * batch_size @@ -234,15 +231,11 @@ class Trainer(object): tensorized_inputs = [] for i in range(len(batch_ques)): - xq = Variable(self.make_input_matrix(batch_ques[i], word_vectors, vec_dim) ) #, requires_grad=False) - xs = Variable(self.make_input_matrix(batch_sents[i], word_vectors, vec_dim) ) #, requires_grad=False) + xq = Variable(self.make_input_matrix(batch_ques[i], word_vectors, vec_dim)) #, requires_grad=False) + xs = Variable(self.make_input_matrix(batch_sents[i], word_vectors, vec_dim)) #, requires_grad=False) ext_feats = Variable(torch.FloatTensor(batch_ext_feats[i])) ext_feats =torch.unsqueeze(ext_feats, 0) y[i] = batch_labels[i] tensorized_inputs.append((xq, xs, ext_feats)) return tensorized_inputs, Variable(y) - - - - diff --git a/sm-model/utils.py b/sm-model/utils.py index 572b3c7..da9fd46 100644 --- a/sm-model/utils.py +++ b/sm-model/utils.py @@ -1,12 +1,11 @@ # file input output -# LATER model input output as well import os import sys -import numpy as np -from gensim.models.keyedvectors import KeyedVectors +import numpy as np import torch +from gensim.models.keyedvectors import KeyedVectors # logging setup import logging @@ -19,7 +18,6 @@ formatter = logging.Formatter('%(levelname)s - %(message)s') ch.setFormatter(formatter) logger.addHandler(ch) - def logargs(func): def inner(*args, **kwargs): logger.info('%s : %s %s' % (func.__name__, args, kwargs)) @@ -29,7 +27,7 @@ def logargs(func): def cache_word_embeddings(word_embeddings_file, cache_file): if not word_embeddings_file.endswith('.gz'): - logger.warning( 'WARNING: expecting a .gz file. Is the {} in the correct format?'.format(word_embeddings_file)) + logger.warning('WARNING: expecting a .gz file. Is the {} in the correct format?'.format(word_embeddings_file)) vocab_size, vec_dim = 0, 0 @@ -39,18 +37,18 @@ def cache_word_embeddings(word_embeddings_file, cache_file): # make cache folder if needed os.mkdir(os.path.dirname(cache_file)) - logger.info( 'caching the word embeddings in np.memmap format' ) + logger.info('caching the word embeddings in np.memmap format') wv = KeyedVectors.load_word2vec_format(word_embeddings_file, binary=True) # print len(wv.syn0), wv.syn0.shape # print len(wv.syn0norm) if wv.syn0norm else None fp = np.memmap(cache_file, dtype=np.double, mode='w+', shape=wv.syn0.shape) fp[:] = wv.syn0[:] with open(cache_file + '.vocab', 'w') as f: - logger.info( 'writing out vocab for {}'.format(word_embeddings_file)) - for _, w in sorted( (voc.index, word) for word, voc in wv.vocab.items()): + logger.info('writing out vocab for {}'.format(word_embeddings_file)) + for _, w in sorted((voc.index, word) for word, voc in wv.vocab.items()): print(w.encode('utf-8'), file=f) with open(cache_file + '.dimensions', 'w') as f: - logger.info( 'writing out dimensions for {}'.format(word_embeddings_file)) + logger.info('writing out dimensions for {}'.format(word_embeddings_file)) print(wv.syn0.shape[0], wv.syn0.shape[1], file=f) vocab_size, vec_dim = wv.syn0.shape del fp, wv @@ -67,7 +65,7 @@ def load_embedding_dimensions(cache_file): def load_cached_embeddings(cache_file, vocab_list, oov_vec = []): - logger.debug( 'loading cached embeddings ') + logger.debug('loading cached embeddings ') with open(cache_file + '.dimensions') as d: vocab_size, vec_dim = [int(e) for e in d.read().strip().split()] @@ -75,7 +73,7 @@ def load_cached_embeddings(cache_file, vocab_list, oov_vec = []): W = np.memmap(cache_file, dtype=np.double, shape=(vocab_size, vec_dim)) with open(cache_file + '.vocab') as f: - logger.debug( 'loading vocab') + logger.debug('loading vocab') w2v_vocab_list = map(str.strip, f.readlines()) vocab_dict = {w:k for k,w in enumerate(w2v_vocab_list)} @@ -101,32 +99,33 @@ def read_in_dataset(dataset_folder, set_folder): max_q = 0 max_s = 0 set_path = os.path.join(dataset_folder, set_folder) - len_q_list =[ len(line.strip().split()) for line in open(os.path.join(set_path, 'a.toks')).readlines() ] - questions = [ line.strip() for line in open(os.path.join(set_path, 'a.toks')).readlines() ] - len_s_list =[ len(line.strip().split()) for line in open(os.path.join(set_path, 'b.toks')).readlines() ] - sentences = [ line.strip() for line in open(os.path.join(set_path, 'b.toks')).readlines() ] - labels = [ int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines() ] - ext_feats = np.array([ list(map(float, line.strip().split(' '))) for line in open(os.path.join(set_path, 'overlap_feats.txt')).readlines() ]) + len_q_list =[len(line.strip().split()) for line in open(os.path.join(set_path, 'a.toks')).readlines()] + questions = [line.strip() for line in open(os.path.join(set_path, 'a.toks')).readlines()] + len_s_list =[len(line.strip().split()) for line in open(os.path.join(set_path, 'b.toks')).readlines()] + sentences = [line.strip() for line in open(os.path.join(set_path, 'b.toks')).readlines()] + labels = [int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines()] + ext_feats = np.array([list(map(float, line.strip().split(' '))) for line in open(os.path.join(set_path, 'overlap_feats.txt')).readlines()]) #y = torch.from_numpy(labels) #return questions, sentences, y - vocab = [ line.strip() for line in open(os.path.join(dataset_folder, 'vocab.txt')).readlines() ] + vocab = [line.strip() for line in open(os.path.join(dataset_folder, 'vocab.txt')).readlines()] return questions, sentences, labels, vocab, max(len_q_list), max(len_s_list), ext_feats + def get_test_qids_labels(dataset_folder, set_folder): set_path = os.path.join(dataset_folder, set_folder) - qids = [ line.strip() for line in open(os.path.join(set_path, 'id.txt')).readlines() ] - labels = np.array([ int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines() ]) + qids = [line.strip() for line in open(os.path.join(set_path, 'id.txt')).readlines()] + labels = np.array([int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines()]) return qids, labels if __name__ == "__main__": - vocab = [ "unk", "idontreallythinkthiswordexists", "hello" ] + vocab = ["unk", "idontreallythinkthiswordexists", "hello"] w2v_dict, vec_dim = load_cached_embeddings("../../data/word2vec-models/aquaint+wiki.txt.gz.ndim=50.cache", vocab) for w, v in w2v_dict.iteritems(): print(w) - print(v) \ No newline at end of file + print(v)