diff --git a/README.md b/README.md index b71b904..6963344 100644 --- a/README.md +++ b/README.md @@ -2,14 +2,14 @@ Pytorch deep learning models. -1. [SM model](./sm-model/README.md): Similarity between question and candidate answers. +1. [SM model](./sm_model/): Similarity between question and candidate answers. ## Setting up Pytorch You need Python 3.6 to use the models in this repository. -As per [pytorch.org](pytorch.org) +As per [pytorch.org](pytorch.org), > [Anaconda](https://www.continuum.io/downloads) is our recommended package manager ```conda install pytorch torchvision -c soumith``` @@ -27,6 +27,6 @@ CUDA installation guide for linux can be found [here](http://docs.nvidia.com/cud **NOTE**: Install CUDA libraries **before** installing conda and pytorch. -## Castor-data +## data for models Sourcing and pre-processing of input data for each model is described in respective ```model/README.md```'s \ No newline at end of file diff --git a/sm-model/SMCastorini.py b/castorini_smmodel_bridge.py similarity index 77% rename from sm-model/SMCastorini.py rename to castorini_smmodel_bridge.py index 5858a24..32c3185 100644 --- a/sm-model/SMCastorini.py +++ b/castorini_smmodel_bridge.py @@ -1,4 +1,5 @@ import os +import sys import pickle import string from collections import defaultdict @@ -8,21 +9,23 @@ import torch from nltk.tokenize import TreebankWordTokenizer from torch.autograd import Variable -from model import QAModel +from sm_model import model + +sys.modules['model'] = model -class SMModelCastorini(object): +class SMModelBridge(object): def __init__(self, model_file, word_embeddings_cache_file, stopwords_file, word2dfs_file): # init torch random seeds torch.manual_seed(1234) - np.random.seed(1234) + np.random.seed(1234) # load model - self.model = QAModel.load('', model_file) + self.model = model.QAModel.load(model_file) # load vectors self.vec_dim = self._preload_cached_embeddings(word_embeddings_cache_file) - self.unk_term_vec = np.random.uniform(-0.25, 0.25, self.vec_dim) + self.unk_term_vec = np.random.uniform(-0.25, 0.25, self.vec_dim) # stopwords self.stoplist = set([line.strip() for line in open(stopwords_file)]) @@ -34,16 +37,16 @@ class SMModelCastorini(object): def _preload_cached_embeddings(self, cache_file): - + with open(cache_file + '.dimensions') as d: vocab_size, vec_dim = [int(e) for e in d.read().strip().split()] self.W = np.memmap(cache_file, dtype=np.double, shape=(vocab_size, vec_dim)) - with open(cache_file + '.vocab') as f: + with open(cache_file + '.vocab') as f: w2v_vocab_list = map(str.strip, f.readlines()) - self.vocab_dict = {w:k for k,w in enumerate(w2v_vocab_list)} + self.vocab_dict = {w:k for k, w in enumerate(w2v_vocab_list)} return vec_dim @@ -64,7 +67,7 @@ class SMModelCastorini(object): feats_overlap = [] for a in a_list: question = q_str.split() - answer = a.split() + answer = a.split() # q_set = set(question) # a_set = set(answer) q_set = set([q for q in question if q not in stoplist]) @@ -82,43 +85,40 @@ class SMModelCastorini(object): df_overlap = 0.0 for w in word_overlap: df_overlap += word2df[w] - + if len(q_set) == 0 and len(a_set) == 0: df_overlap = 0 else: df_overlap /= (len(q_set) + len(a_set)) - feats_overlap.append(np.array([ - overlap, - df_overlap, - ])) + feats_overlap.append(np.array([overlap, df_overlap])) return np.array(feats_overlap) - def make_input_matrix(self, sentence): - terms = sentence.strip().split() + def make_input_matrix(self, sentence): + terms = sentence.strip().split() # word_embeddings = torch.zeros(max_len, vec_dim).type(torch.DoubleTensor) word_embeddings = torch.zeros(len(terms), self.vec_dim).type(torch.DoubleTensor) for i in range(len(terms)): - word = terms[i] + word = terms[i] if word not in self.vocab_dict: emb = torch.from_numpy(self.unk_term_vec) else: - emb = torch.from_numpy(self.W[self.vocab_dict[word]]) - word_embeddings[i] = emb + emb = torch.from_numpy(self.W[self.vocab_dict[word]]) + word_embeddings[i] = emb input_tensor = torch.zeros(1, self.vec_dim, len(terms)) input_tensor[0] = torch.transpose(word_embeddings, 0, 1) return input_tensor - def get_tensorized_inputs(self, batch_ques, batch_sents, batch_ext_feats): - assert(1 == len(batch_ques)) + def get_tensorized_inputs(self, batch_ques, batch_sents, batch_ext_feats): + assert(1 == len(batch_ques)) tensorized_inputs = [] for i in range(len(batch_ques)): - xq = Variable(self.make_input_matrix(batch_ques[i])) + xq = Variable(self.make_input_matrix(batch_ques[i])) xs = Variable(self.make_input_matrix(batch_sents[i])) ext_feats = Variable(torch.FloatTensor(batch_ext_feats[i])) - ext_feats =torch.unsqueeze(ext_feats, 0) + ext_feats = torch.unsqueeze(ext_feats, 0) tensorized_inputs.append((xq, xs, ext_feats)) return tensorized_inputs @@ -128,30 +128,33 @@ class SMModelCastorini(object): q_str, a_list = self.parser(question, answers) # calculate overlap features - overlap_feats = self.compute_overlap_features(q_str,a_list, stoplist=None, word2df=self.word2dfs) - overlap_feats_stoplist = self.compute_overlap_features(q_str, a_list, stoplist=self.stoplist, word2df=self.word2dfs) + overlap_feats = self.compute_overlap_features(q_str, a_list, \ + stoplist=None, word2df=self.word2dfs) + overlap_feats_stoplist = self.compute_overlap_features(q_str, a_list, \ + stoplist=self.stoplist, word2df=self.word2dfs) overlap_feats_vec = np.hstack([overlap_feats, overlap_feats_stoplist]) # run through the model scores_sentences = [] for i in range(len(a_list)): - xq, xa, x_ext_feats = self.get_tensorized_inputs([q], [a_list[i]], [overlap_feats_vec[i]])[0] - pred = self.model(xq, xa, x_ext_feats) + xq, xa, x_ext_feats = self.get_tensorized_inputs([q_str], [a_list[i]], \ + [overlap_feats_vec[i]])[0] + pred = self.model(xq, xa, x_ext_feats) pred = torch.exp(pred) scores_sentences.append((pred.data.squeeze()[1], a_list[i])) - + return scores_sentences if __name__ == "__main__": - - smmodel = SMModelCastorini('sm.model.py3', - '../../data/word2vec-models/aquaint+wiki.txt.gz.ndim=50.cache', - 'stopwords.txt', - 'word2dfs.p') - - q = "who is the author of the book , `` the iron lady : a biography of margaret thatcher '' ?" - a = [ + + smmodel = SMModelBridge('../models/sm_model/sm_model.TrecQA.TRAIN-ALL.2017-04-02.castor', + '../data/word2vec/aquaint+wiki.txt.gz.ndim=50.cache', + '../data/TrecQA/stopwords.txt', + '../data/TrecQA/word2dfs.p') + + question = "who is the author of the book , `` the iron lady : a biography of margaret thatcher '' ?" + answers = [ "the iron lady ; a biography of margaret thatcher by hugo young -lrb- farrar , straus & giroux -rrb-", "in this same revisionist mold , hugo young , the distinguished british journalist , has performed a brilliant \ dissection of the notion of thatcher as a conservative icon .", @@ -163,8 +166,8 @@ if __name__ == "__main__": "this is not the answer", "asdfawe asdf sertse dgfsgsfg" ] - - ss = smmodel.rerank_candidate_answers(q, a) - print('Question:', q) + + ss = smmodel.rerank_candidate_answers(question, answers) + print('Question:', question) for score, sentence in ss: print(score, '\t', sentence) diff --git a/sm-model/README.md b/sm-model/README.md deleted file mode 100644 index 2455be0..0000000 --- a/sm-model/README.md +++ /dev/null @@ -1,33 +0,0 @@ -## Similarity Measure model (SM model) - -#### References: -1. Aliaksei Severyn and Alessandro Moschitti. 2015. Learning to Rank Short Text Pairs with Convolutional Deep Neural Networks. In Proceedings of the 38th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '15). ACM, New York, NY, USA, 373-382. DOI: http://dx.doi.org/10.1145/2766462.2767738 - - -#### TODOs: -1. figure out if the L2 regularization is correct -2. Batch size of 50 (current batch_size = 1) - -#### Getting the data - -TODO: - -#### Running it - -``1.`` Make TrecEval: -``` -$ cd trec_eval-8.0 -$ make clean -$ make -``` - -``2.`` Get the Overlapping features for Q and A: -``` -$ python overlap_features.py ../../data/TrecQA -``` - -``3.`` To run the S&M model on TrecQA, please follow the same parameter setting: -``` -$ python main.py ../../data/aquaint+wiki.txt.gz.ndim\=50.bin ../../data/TrecQA/ sm --batch_size 1 -``` - diff --git a/sm-model/overlap_features.py b/sm-model/overlap_features.py deleted file mode 100644 index 3dd68bf..0000000 --- a/sm-model/overlap_features.py +++ /dev/null @@ -1,190 +0,0 @@ -import sys -import re -import os -import numpy as np - -import argparse - -#from nltk.corpus import stopwords -from nltk.stem.porter import * -from collections import defaultdict - -def load_data(dname): - stemmer = PorterStemmer() - qids, questions, answers, labels = [], [], [], [] - print(dname) - with open(dname+'a.toks') as f: - for line in f: - line = unicode(line, errors='ignore') - question = line.strip().split() - question = [stemmer.stem(word) for word in question] - questions.append(question) - with open(dname+'b.toks') as f: - for line in f: - line = unicode(line, errors='ignore') - answer = line.decode('utf-8').strip().split() - answer = [stemmer.stem(word) for word in answer] - answers.append(answer) - with open(dname+'id.txt') as f: - for line in f: - qids.append(line.strip()) - with open(dname+'sim.txt') as f: - for line in f: - labels.append(int(line.strip())) - return qids, questions, answers, labels - -def compute_overlap_features(questions, answers, word2df=None, stoplist=None): - word2df = word2df if word2df else {} - stoplist = stoplist if stoplist else set() - feats_overlap = [] - for question, answer in zip(questions, answers): - # q_set = set(question) - # a_set = set(answer) - q_set = set([q for q in question if q not in stoplist]) - a_set = set([a for a in answer if a not in stoplist]) - word_overlap = q_set.intersection(a_set) - # overlap = float(len(word_overlap)) / (len(q_set) * len(a_set) + 1e-8) - if len(q_set) == 0 and len(a_set) == 0: - overlap = 0 - else: - overlap = float(len(word_overlap)) / (len(q_set) + len(a_set)) - - # q_set = set([q for q in question if q not in stoplist]) - # a_set = set([a for a in answer if a not in stoplist]) - word_overlap = q_set.intersection(a_set) - df_overlap = 0.0 - for w in word_overlap: - df_overlap += word2df[w] - if len(q_set) == 0 and len(a_set) == 0: - df_overlap = 0 - else: - df_overlap /= (len(q_set) + len(a_set)) - - feats_overlap.append(np.array([ - overlap, - df_overlap, - ])) - return np.array(feats_overlap) - -def compute_overlap_idx(questions, answers, stoplist, q_max_sent_length, a_max_sent_length): - stoplist = stoplist if stoplist else [] - feats_overlap = [] - q_indices, a_indices = [], [] - for question, answer in zip(questions, answers): - q_set = set([q for q in question if q not in stoplist]) - a_set = set([a for a in answer if a not in stoplist]) - word_overlap = q_set.intersection(a_set) - - q_idx = np.ones(q_max_sent_length) * 2 - for i, q in enumerate(question): - value = 0 - if q in word_overlap: - value = 1 - q_idx[i] = value - q_indices.append(q_idx) - - #### ERROR - # a_idx = np.ones(a_max_sent_length) * 2 - # for i, q in enumerate(question): - # value = 0 - # if q in word_overlap: - - a_idx = np.ones(a_max_sent_length) * 2 - for i, a in enumerate(answer): - value = 0 - if a in word_overlap: - value = 1 - a_idx[i] = value - a_indices.append(a_idx) - - q_indices = np.vstack(q_indices).astype('int32') - a_indices = np.vstack(a_indices).astype('int32') - - return q_indices, a_indices - -def compute_dfs(docs): - word2df = defaultdict(float) - for doc in docs: - for w in set(doc): - word2df[w] += 1.0 - num_docs = len(docs) - for w, value in word2df.iteritems(): - word2df[w] /= np.math.log(num_docs / value) - return word2df - -if __name__ == '__main__': - ap = argparse.ArgumentParser(description="compute overlap features for SM model") - ap.add_argument("dataset", help="path/to/dataset-directory", default="../../data/TrecQA") - ap.add_argument("--train_all", help="will generate overlap features for the train-all dataset", action="store_true") - args = ap.parse_args() - - stoplist = set([line.strip() for line in open('stopwords.txt')]) - import string - punct = set(string.punctuation) - stoplist.update(punct) - #stoplist = None - - all_questions, all_answers, all_qids = [], [], [] - base_dir = args.dataset - # base_dir = '../../data/' + sys.argv[1] + '/' - # sub_dirs = ['train/', 'raw-dev/', 'test.minimal/','test.complete/'] - # sub_dirs = ['train-all/', 'raw-dev/', 'raw-test/'] - sub_dirs = ['train/', 'clean-dev/', 'clean-test/'] - if args.train_all: - sub_dirs = ['train-all/', 'raw-dev/', 'raw-test/'] - - for sub in sub_dirs: - qids, questions, answers, labels = load_data(base_dir+sub) - all_questions.extend(questions) - all_answers.extend(answers) - all_qids.extend(qids) - - seen = set() - unique_questions = [] - for q, qid in zip(all_questions, all_qids): - if qid not in seen: - seen.add(qid) - unique_questions.append(q) - - docs = all_answers + unique_questions - word2dfs = compute_dfs(docs) - print(word2dfs.items()[:10]) - - q_max_sent_length = max(map(lambda x: len(x), all_questions)) - a_max_sent_length = max(map(lambda x: len(x), all_answers)) - print('q_max_sent_length', q_max_sent_length) - print('a_max_sent_length', a_max_sent_length) - - for sub in sub_dirs: - print(sub) - qids, questions, answers, labels = load_data(base_dir+sub) - - overlap_feats = compute_overlap_features(questions, answers, stoplist=None, word2df=word2dfs) - overlap_feats_stoplist = compute_overlap_features(questions, answers, stoplist=stoplist, word2df=word2dfs) - overlap_feats = np.hstack([overlap_feats, overlap_feats_stoplist]) - print(overlap_feats[:3]) - print('overlap_feats', overlap_feats.shape) - - from sklearn.preprocessing import StandardScaler - scaler = StandardScaler() - print("Scaling overlap features") - overlap_feats = scaler.fit_transform(overlap_feats) - print(overlap_feats[:3]) - - '''q_overlap_indices, a_overlap_indices = compute_overlap_idx(questions, answers, stoplist, q_max_sent_length, a_max_sent_length) - print(q_overlap_indices[:3], a_overlap_indices[:3]) - print('q_overlap_indices', q_overlap_indices.shape) - print('a_overlap_indices', a_overlap_indices.shape)''' - - with open(base_dir+sub+'overlap_feats.txt', 'w') as f: - for i in range(overlap_feats.shape[0]): - for j in range(4): - f.write(str(overlap_feats[i][j]) + ' ') - f.write('\n') - '''with open(base_dir+sub+'overlap_indices.txt', 'w') as f: - for i in range(q_overlap_indices.shape[0]): - for j in range(q_max_sent_length): - f.write(str(q_overlap_indices[i][j]) + ' ') - for j in range(a_max_sent_length): - f.write(str(a_overlap_indices[i][j]) + ' ') - f.write('\n')''' diff --git a/sm-model/stopwords.txt b/sm-model/stopwords.txt deleted file mode 100644 index 5c78865..0000000 --- a/sm-model/stopwords.txt +++ /dev/null @@ -1,238 +0,0 @@ -'s -I -a -aboard -about -above -across -after -afterwards -against -agin -ago -agreed-upon -ah -alas -albeit -all -all-over -almost -along -alongside -altho -although -amid -amidst -among -amongst -an -and -another -any -anyone -anything -around -as -aside -astride -at -atop -avec -away -back -be -because -before -beforehand -behind -behynde -below -beneath -beside -besides -between -bewteen -beyond -bi -both -but -by -ca. -de -des -despite -do -down -due -durin -during -each -eh -either -en -every -ever -everyone -everything -except -far -fer -for -from -go -goddamn -goody -gosh -half -have -he -hell -her -herself -hey -him -himself -his -ho -how -however -i -if -in -inside -insofar -instead -into -it -its -itself -la -le -les -lest -lieu -like -me -minus -moreover -my -myself -near -near-by -nearer -nearest -neither -nevertheless -next -no -nor -not -nothing -notwithstanding -o -o'er -of -off -on -once -one -oneself -only -onto -or -other -others -otherwise -our -ours -ourselves -out -outside -outta -over -per -rather -regardless -round -se -she -should -since -so -some -someone -something -than -that -the -their -them -themselves -then -there -therefore -these -they -thine -this -those -thou -though -through -throughout -thru -till -to -together -toward -towardes -towards -uh -under -underneath -unless -unlike -until -unto -up -upon -uppon -us -via -vis-a-vis -vis-�-vis -we -well -what -whatever -whatsoever -when -whenever -where -whereas -wherefore -whereupon -whether -which -whichever -while -who -whoever -whom -whose -why -with -withal -within -without -ye -yea -yeah -yes -yet -yonder -you -your -yours -yourself -yourselves diff --git a/sm-model/.gitignore b/sm_model/.gitignore similarity index 100% rename from sm-model/.gitignore rename to sm_model/.gitignore diff --git a/sm_model/README.md b/sm_model/README.md new file mode 100644 index 0000000..07bef40 --- /dev/null +++ b/sm_model/README.md @@ -0,0 +1,48 @@ +## SM model + +#### References: +1. Aliaksei _S_everyn and Alessandro _M_oschitti. 2015. Learning to Rank Short Text Pairs with Convolutional Deep Neural Networks. In Proceedings of the 38th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '15). ACM, New York, NY, USA, 373-382. DOI: http://dx.doi.org/10.1145/2766462.2767738 + + +#### TODOs: +1. figure out if the L2 regularization is correct +2. Batch size of 50 (current batch_size = 1) + +#### Requirements +gensim==1.0.1 +nltk==3.2.2 +numpy==1.11.3 +pandas==0.19.2 +torch==0.1.11+b13b701 + +#### Getting the data + +git clone [castorini/data](https://github.com/castorini/data) + +castorini/data contains: + +```word2vec/aquaint+wiki.txt.gz.ndim=50.bin```: word embeddings. +Note that a memory mapped cache will be created on first use on your disk, when you run ```main.py``` (below). + +```TrecQA/```: the directory with the input data for training the model. + +Follow instructions in castorini/data/TrecQA/README.md to preprocess data for it to be ingestable by the model. + + +#### Running the model + +``1.`` Make TrecEval: +``` +$ cd trec_eval-8.0 +$ make clean +$ make +``` + +``2.`` To run the S&M model on TrecQA, please follow the same parameter setting: +``` +$ python main.py ../../model/sm.model.aquaint.train-all --train_all +``` +The final model will be saved to ```../../model/sm.model.aquaint.train-all``` + +Run ```python main.py -h``` for more default options. + diff --git a/sm_model/__init__.py b/sm_model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sm-model/main.py b/sm_model/main.py similarity index 70% rename from sm-model/main.py rename to sm_model/main.py index d0f8b3b..6dc2063 100644 --- a/sm-model/main.py +++ b/sm_model/main.py @@ -8,13 +8,10 @@ import sys import numpy as np import pandas as pd import torch -import torch.nn as nn -import torch.optim as optim -from torch.autograd import Variable import utils -from model import QAModel from train import Trainer +from model import QAModel # logging setup import logging @@ -41,14 +38,16 @@ def compute_map_mrr(dataset_folder, set_folder, test_scores): qids_test, y_test = utils.get_test_qids_labels(dataset_folder, set_folder) # Call TrecEval code to calc MAP and MRR - df_submission = pd.DataFrame(index=np.arange(N), columns=['qid', 'iter', 'docno', 'rank', 'sim', 'run_id']) + df_submission = pd.DataFrame(index=np.arange(N), \ + columns=['qid', 'iter', 'docno', 'rank', 'sim', 'run_id']) df_submission['qid'] = qids_test df_submission['iter'] = 0 df_submission['docno'] = np.arange(N) df_submission['rank'] = 0 df_submission['sim'] = test_scores df_submission['run_id'] = 'smmodel' - df_submission.to_csv(os.path.join(args.dataset_folder, 'submission.txt'), header=False, index=False, sep=' ') + df_submission.to_csv(os.path.join(args.dataset_folder, 'submission.txt'), \ + header=False, index=False, sep=' ') df_gold = pd.DataFrame(index=np.arange(N), columns=['qid', 'iter', 'docno', 'rel']) df_gold['qid'] = qids_test @@ -60,8 +59,8 @@ def compute_map_mrr(dataset_folder, set_folder, test_scores): # subprocess.call("/bin/sh run_eval.sh '{}'".format(args.dataset_folder), shell=True) pargs = shlex.split("/bin/sh run_eval.sh '{}'".format(args.dataset_folder)) p = subprocess.Popen(pargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - pout, perr = p.communicate() - + pout, perr = p.communicate() + lines = pout.split(b'\n') map = float(lines[0].strip().split()[-1]) mrr = float(lines[1].strip().split()[-1]) @@ -70,33 +69,46 @@ def compute_map_mrr(dataset_folder, set_folder, test_scores): if __name__ == "__main__": - ap = argparse.ArgumentParser(description='pytorch port of the SM model') - ap.add_argument('word_vectors_file', help='NOTE: a cache will be created for faster loading for word vectors') - ap.add_argument('dataset_folder', help='directory containing train, dev, test sets') - ap.add_argument('model_fname', help='model will be saved in args.dataset_folder/') + ap = argparse.ArgumentParser(description='pytorch port of the SM model', \ + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + + ap.add_argument('model_outfile', help='file to save final model') + + ap.add_argument('--word_vectors_file', \ + help='NOTE: a cache will be created for faster loading for word vectors', + default="../../data/word2vec/aquaint+wiki.txt.gz.ndim=50.bin") + ap.add_argument('--dataset_folder', help='directory containing train, dev, test sets', \ + default="../../data/TrecQA") + ap.add_argument('--classes', type=int, default=2) # system arguments # TODO: add arguments for CUDA - ap.add_argument('--num_threads', help="the number of simultaneous processes to run", type=int, default=4) + ap.add_argument('--num_threads', help="the number of simultaneous processes to run", \ + type=int, default=4) # training arguments - ap.add_argument('--batch_size', type=int, default=1) - ap.add_argument('--filter_width', type=int, default=5) + ap.add_argument('--batch_size', type=int, default=1, help="training mini-batch size") + ap.add_argument('--filter_width', type=int, default=5, help="number of convolution channels") ap.add_argument('--eta', help='Initial learning rate', default=0.001, type=float) - ap.add_argument('--mom', help='SGD Momentum', default=0.0, type=float) + ap.add_argument('--mom', help='SGD Momentum', default=0.0, type=float) ap.add_argument('--train_all', help='switches to train-all set', action="store_true") - + # epoch related arguments - ap.add_argument('--epochs', type=int, default=25) - ap.add_argument('--patience', type=int, default=5, help="if there is no appreciable change in model after epochs, then stop") - + ap.add_argument('--epochs', type=int, default=25, help="number of trainin epochs") + ap.add_argument('--patience', type=int, default=5, \ + help="if there is no appreciable change in model after epochs, then stop") + # debugging arguments - ap.add_argument('--debugSingleBatch', action="store_true", help="will stop program after training 1 input batch") - ap.add_argument('--num_conv_filters', help="the number of convolution channels (lesser is faster)", default=100, type=int) - ap.add_argument('--no_ext_feats', action="store_true", help="will not include external features in the model") + ap.add_argument('--debug_single_batch', action="store_true", \ + help="will stop program after training 1 input batch") + ap.add_argument('--num_conv_filters', default=100, type=int, \ + help="the number of convolution channels (lesser is faster)") + ap.add_argument('--no_ext_feats', action="store_true", \ + help="will not include external features in the model") ap.add_argument('--no_loss_reg', help="no loss regularization", action="store_true") - ap.add_argument('--test_on_each_epoch', help='runs test on each epoch to track final performance', action="store_true") + ap.add_argument('--test_on_each_epoch', action="store_true", \ + help='runs test on each epoch to track final performance') args = ap.parse_args() @@ -108,17 +120,17 @@ if __name__ == "__main__": train_set, dev_set, test_set = 'train-all', 'raw-dev', 'raw-test' # cache word embeddings - cache_file = os.path.splitext(args.word_vectors_file)[0] + '.cache' + cache_file = os.path.splitext(args.word_vectors_file)[0] + '.cache' utils.cache_word_embeddings(args.word_vectors_file, cache_file) vocab_size, vec_dim = utils.load_embedding_dimensions(cache_file) - + # instantiate model - net = QAModel(vec_dim, args.filter_width, args.num_conv_filters, args.no_ext_feats) #filter width is 5 - QAModel.save(net, args.dataset_folder, args.model_fname) + net = QAModel(vec_dim, args.filter_width, args.num_conv_filters, args.no_ext_feats) + QAModel.save(net, args.model_outfile) torch.set_num_threads(args.num_threads) - + trainer = Trainer(net, args.eta, args.mom, args.no_loss_reg, vec_dim) logger.info("Loading input data...") trainer.load_input_data(args.dataset_folder, cache_file, train_set, dev_set, test_set) @@ -127,38 +139,38 @@ if __name__ == "__main__": best_model = 0 for i in range(args.epochs): - logger.info('------------- Training epoch {} --------------'.format(i+1)) - train_accuracy = trainer.train(train_set, args.batch_size, args.debugSingleBatch) - if args.debugSingleBatch: sys.exit(0) - + logger.info('------------- Training epoch {} --------------'.format(i+1)) + train_accuracy = trainer.train(train_set, args.batch_size, args.debug_single_batch) + if args.debug_single_batch: sys.exit(0) + dev_scores = trainer.test(dev_set, args.batch_size) - + dev_map, dev_mrr = compute_map_mrr(args.dataset_folder, dev_set, dev_scores) logger.info("------- MAP {}, MRR {}".format(dev_map, dev_mrr)) - if dev_map - best_map > 1e-3: # new map is better than best map + if dev_map - best_map > 1e-3: # new map is better than best map best_model = i best_map = dev_map - QAModel.save(net, args.dataset_folder, args.model_fname) + QAModel.save(net, args.model_outfile) logger.info('Achieved better dev_map ... saved model') - if args.test_on_each_epoch: - test_scores = trainer.test(test_set, args.batch_size) + if args.test_on_each_epoch: + test_scores = trainer.test(test_set, args.batch_size) map, mrr = compute_map_mrr(args.dataset_folder, test_set, test_scores) logger.info("------- MAP {}, MRR {}".format(map, mrr)) - + if (i - best_model) >= args.patience: logger.warning('No improvement since the last {} epochs. Stopping training'.format(i - best_model)) break - - logger.info(' ------------ Training epochs completed! ------------') + + logger.info(' ------------ Training epochs completed! ------------') logger.info('Best MAP in training phase = {:.4f}'.format(best_map)) - trained_model = QAModel.load(args.dataset_folder, args.model_fname) - evaluator = Trainer(trained_model, args.eta, args.mom, args.no_loss_reg, vec_dim) + trained_model = QAModel.load(args.model_outfile) + evaluator = Trainer(trained_model, args.eta, args.mom, args.no_loss_reg, vec_dim) evaluator.load_input_data(args.dataset_folder, cache_file, None, None, test_set) test_scores = evaluator.test(test_set, args.batch_size) - + map, mrr = compute_map_mrr(args.dataset_folder, test_set, test_scores) logger.info("------- MAP {}, MRR {}".format(map, mrr)) diff --git a/sm-model/model.py b/sm_model/model.py similarity index 66% rename from sm-model/model.py rename to sm_model/model.py index 22f1207..a820938 100644 --- a/sm-model/model.py +++ b/sm_model/model.py @@ -1,10 +1,6 @@ -import os - -import numpy as np import torch import torch.nn as nn import torch.nn.functional as F -from torch.autograd import Variable # logging setup import logging @@ -19,18 +15,19 @@ logger.addHandler(ch) class QAModel(nn.Module): - - @staticmethod - def save(model, out_folder, model_fname): - torch.save(model, os.path.join(out_folder, model_fname)) - @staticmethod - def load(in_folder, model_fname): - return torch.load(os.path.join(in_folder, model_fname)) - - - def __init__(self, input_n_dim, filter_width, conv_filters=100, no_ext_feats=False, ext_feats_size=4, n_classes=2): + def save(model, model_fname): + torch.save(model, model_fname) + + + @staticmethod + def load(model_fname): + return torch.load(model_fname) + + + def __init__(self, input_n_dim, filter_width, \ + conv_filters=100, no_ext_feats=False, ext_feats_size=4, n_classes=2): super(QAModel, self).__init__() self.no_ext_feats = no_ext_feats @@ -40,25 +37,26 @@ class QAModel(nn.Module): self.conv_q = nn.Sequential( nn.Conv1d(input_n_dim, self.conv_channels, filter_width, padding=filter_width-1), - nn.Tanh() + nn.Tanh() ) self.conv_a = nn.Sequential( nn.Conv1d(input_n_dim, self.conv_channels, filter_width, padding=filter_width-1), - nn.Tanh() + nn.Tanh() ) - - self.combined_feature_vector = nn.Linear(2*self.conv_channels + (0 if no_ext_feats else ext_feats_size), n_hidden) - #TODO: add +1 to Linear layer^. Will need change in forward function + + self.combined_feature_vector = nn.Linear(2*self.conv_channels + \ + (0 if no_ext_feats else ext_feats_size), n_hidden) + # TODO: add +1 to Linear layer^. Will need change in forward function self.combined_features_activation = nn.Tanh() self.dropout = nn.Dropout(0.5) self.hidden = nn.Linear(n_hidden, n_classes) self.logsoftmax = nn.LogSoftmax() - - def forward(self, question, answer, ext_feats): - - q = self.conv_q.forward(question) + + def forward(self, question, answer, ext_feats): + + q = self.conv_q.forward(question) q = F.max_pool1d(q, q.size()[2]) q = q.view(-1, self.conv_channels) # logger.debug('forward q: {}'.format(q)) @@ -69,12 +67,12 @@ class QAModel(nn.Module): x = None if self.no_ext_feats: - x = torch.cat([q, a], 1) + x = torch.cat([q, a], 1) # logger.debug('no_ext_feats') else: - x = torch.cat([q, a, ext_feats], 1) + x = torch.cat([q, a, ext_feats], 1) # logger.debug('with ext_feats') - + # logger.debug('featvec x: {}'.format(x)) # logger.debug(x.creator) diff --git a/sm-model/run_eval.sh b/sm_model/run_eval.sh similarity index 100% rename from sm-model/run_eval.sh rename to sm_model/run_eval.sh diff --git a/sm-model/train.py b/sm_model/train.py similarity index 71% rename from sm-model/train.py rename to sm_model/train.py index 9579850..3ce3270 100644 --- a/sm-model/train.py +++ b/sm_model/train.py @@ -1,5 +1,3 @@ -import argparse - import time import numpy as np @@ -23,71 +21,75 @@ logger.addHandler(ch) class Trainer(object): - + def __init__(self, model, eta, mom, no_loss_reg, vec_dim): - # set the random seeds for every instance of trainer. + # set the random seeds for every instance of trainer. # needed to ensure reproduction of random word vectors for out of vocab terms torch.manual_seed(1234) np.random.seed(1234) - self.unk_term = np.random.uniform(-0.25, 0.25, vec_dim) + self.unk_term = np.random.uniform(-0.25, 0.25, vec_dim) self.reg = 1e-5 self.no_loss_reg = no_loss_reg self.model = model self.criterion = nn.CrossEntropyLoss() #self.criterion = nn.NLLLoss() - self.optimizer = optim.SGD(self.model.parameters(), lr=eta, momentum=mom, weight_decay=(0 if no_loss_reg else self.reg)) + self.optimizer = optim.SGD(self.model.parameters(), lr=eta, momentum=mom, \ + weight_decay=(0 if no_loss_reg else self.reg)) self.datasets = {} self.embeddings = {} self.vec_dim = vec_dim - - def load_input_data(self, dataset_root_folder, word_vectors_cache_file, train_set_folder, dev_set_folder, test_set_folder): + + def load_input_data(self, dataset_root_folder, word_vectors_cache_file, \ + train_set_folder, dev_set_folder, test_set_folder): for set_folder in [test_set_folder, dev_set_folder, train_set_folder]: if set_folder: self.datasets[set_folder] = utils.read_in_dataset(dataset_root_folder, set_folder) - # NOTE: self.datasets[set_folder] = questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats - self.embeddings[set_folder] = utils.load_cached_embeddings(word_vectors_cache_file, - self.datasets[set_folder][3], [] if "train" in set_folder else self.unk_term) - - - def regularize_loss(self, loss): - + # NOTE: self.datasets[set_folder] = questions, sentences, labels, + # vocab, maxlen_q, maxlen_s, ext_feats + self.embeddings[set_folder] = utils.load_cached_embeddings( \ + word_vectors_cache_file, self.datasets[set_folder][3], \ + [] if "train" in set_folder else self.unk_term) + + + def regularize_loss(self, loss): + flattened_params = [] for p in self.model.parameters(): f = p.data.clone() flattened_params.append(f.view(-1)) - + fp = torch.cat(flattened_params) loss = loss + 0.5 * self.reg * fp.norm() * fp.norm() # for p in self.model.parameters(): # loss = loss + 0.5 * self.reg * p.norm() * p.norm() - - return loss - + return loss + + def _train(self, xq, xa, ext_feats, ys): - - self.optimizer.zero_grad() - output = self.model(xq, xa, ext_feats) - loss = self.criterion(output, ys) + + self.optimizer.zero_grad() + output = self.model(xq, xa, ext_feats) + loss = self.criterion(output, ys) # logger.debug('loss after criterion {}'.format(loss)) # NOTE: regularizing location 1 if not self.no_loss_reg: - loss = self.regularize_loss(loss) + loss = self.regularize_loss(loss) # logger.debug('loss after regularizing {}'.format(loss)) - + loss.backward() - + # logger.debug('AFTER backward') #logger.debug('params {}'.format([p for p in self.model.parameters()])) # logger.debug('params grads {}'.format([p.grad for p in self.model.parameters()])) - + # NOTE: regularizing location 2. It would seem that location 1 is correct? #if not self.no_loss_reg: # loss = self.regularize_loss(loss) @@ -102,18 +104,19 @@ class Trainer(object): return loss.data[0], self.pred_equals_y(output, ys) - def pred_equals_y(self, pred, y): - _, best = pred.max(1) - best = best.data.long().squeeze() + def pred_equals_y(self, pred, y): + _, best = pred.max(1) + best = best.data.long().squeeze() return torch.sum(y.data.long() == best) def test(self, set_folder, batch_size): logger.info('----- Predictions on {} '.format(set_folder)) - - questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = self.datasets[set_folder] + + questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = \ + self.datasets[set_folder] word_vectors, vec_dim = self.embeddings[set_folder], self.vec_dim - + self.model.eval() batch_size = 1 @@ -123,31 +126,32 @@ class Trainer(object): num_batches = np.ceil(len(questions)/batch_size) y_pred = np.zeros(len(questions)) ypc = 0 - + for k in range(int(num_batches)): batch_start = k * batch_size batch_end = (k+1) * batch_size # convert raw questions and sentences to tensors batch_inputs, batch_labels = self.get_tensorized_inputs( - questions[batch_start:batch_end], - sentences[batch_start:batch_end], - labels[batch_start:batch_end], - ext_feats[batch_start:batch_end], - word_vectors, vocab, vec_dim + questions[batch_start:batch_end], + sentences[batch_start:batch_end], + labels[batch_start:batch_end], + ext_feats[batch_start:batch_end], + word_vectors, vocab, vec_dim ) - + xq, xa, x_ext_feats = batch_inputs[0] y = batch_labels[0] - - pred = self.model(xq, xa, x_ext_feats) - loss = self.criterion(pred, y) + + pred = self.model(xq, xa, x_ext_feats) + loss = self.criterion(pred, y) pred = torch.exp(pred) total_loss += loss # total_correct += self.pred_equals_y(pred, y) - y_pred[ypc] = pred.data.squeeze()[1] # we want to score for relevance, NOT the predicted class - ypc += 1 - + y_pred[ypc] = pred.data.squeeze()[1] + # ^ we want to score for relevance, NOT the predicted class + ypc += 1 + # logger.info('{}_correct {}'.format(set_folder, total_correct)) # logger.info('{}_loss {}'.format(set_folder, total_loss.data[0])) logger.info('{} total {}'.format(set_folder, len(labels))) @@ -157,12 +161,13 @@ class Trainer(object): return y_pred - def train(self, set_folder, batch_size, debugSingleBatch): + def train(self, set_folder, batch_size, debug_single_batch): train_start_time = time.time() - questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = self.datasets[set_folder] + questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = \ + self.datasets[set_folder] word_vectors, vec_dim = self.embeddings[set_folder], self.vec_dim - + # set model for training modep self.model.train() @@ -175,23 +180,24 @@ class Trainer(object): # convert raw questions and sentences to tensors batch_inputs, batch_labels = self.get_tensorized_inputs( - questions[batch_start:batch_end], - sentences[batch_start:batch_end], - labels[batch_start:batch_end], - ext_feats[batch_start:batch_end], - word_vectors, vocab, vec_dim + questions[batch_start:batch_end], + sentences[batch_start:batch_end], + labels[batch_start:batch_end], + ext_feats[batch_start:batch_end], + word_vectors, vocab, vec_dim ) - + xq, xa, x_ext_feats = batch_inputs[0] ys = batch_labels[0] - batch_loss, batch_correct = self._train(xq, xa, x_ext_feats, ys) - + batch_loss, batch_correct = self._train(xq, xa, x_ext_feats, ys) + # logger.debug('batch_loss {}, batch_correct {}'.format(batch_loss, batch_correct)) train_loss += batch_loss # train_correct += batch_correct - if debugSingleBatch: break + if debug_single_batch: + break # logger.info('train_correct {}'.format(train_correct)) logger.info('train_loss {}'.format(train_loss)) @@ -201,23 +207,25 @@ class Trainer(object): )) logger.info('training time = {:.3f} seconds'.format(time.time() - train_start_time)) return train_correct/num_batches - - def make_input_matrix(self, sentence, word_vectors, vec_dim): - terms = sentence.strip().split() - # word_embeddings = torch.zeros(max_len, vec_dim).type(torch.DoubleTensor) + + def make_input_matrix(self, sentence, word_vectors, vec_dim): + terms = sentence.strip().split()[:60] + # NOTE: we are truncating the inputs to 60 words. + word_embeddings = torch.zeros(len(terms), vec_dim).type(torch.DoubleTensor) for i in range(len(terms)): - word = terms[i] - emb = torch.from_numpy(word_vectors[word]) - word_embeddings[i] = emb + word = terms[i] + emb = torch.from_numpy(word_vectors[word]) + word_embeddings[i] = emb input_tensor = torch.zeros(1, vec_dim, len(terms)) - input_tensor[0] = torch.transpose(word_embeddings, 0 , 1) + input_tensor[0] = torch.transpose(word_embeddings, 0, 1) return input_tensor - def get_tensorized_inputs(self, batch_ques, batch_sents, batch_labels, batch_ext_feats, word_vectors, vocab, vec_dim): + def get_tensorized_inputs(self, batch_ques, batch_sents, batch_labels, batch_ext_feats, \ + word_vectors, vocab, vec_dim): batch_size = len(batch_ques) # NOTE: ideal batch size is one, because sentences are all of different length. # In other words, we have no option but to feed in sentences one by one into the model @@ -231,10 +239,10 @@ class Trainer(object): tensorized_inputs = [] for i in range(len(batch_ques)): - xq = Variable(self.make_input_matrix(batch_ques[i], word_vectors, vec_dim)) #, requires_grad=False) - xs = Variable(self.make_input_matrix(batch_sents[i], word_vectors, vec_dim)) #, requires_grad=False) + xq = Variable(self.make_input_matrix(batch_ques[i], word_vectors, vec_dim)) + xs = Variable(self.make_input_matrix(batch_sents[i], word_vectors, vec_dim)) ext_feats = Variable(torch.FloatTensor(batch_ext_feats[i])) - ext_feats =torch.unsqueeze(ext_feats, 0) + ext_feats = torch.unsqueeze(ext_feats, 0) y[i] = batch_labels[i] tensorized_inputs.append((xq, xs, ext_feats)) diff --git a/sm-model/trec_eval-8.0/Makefile b/sm_model/trec_eval-8.0/Makefile similarity index 100% rename from sm-model/trec_eval-8.0/Makefile rename to sm_model/trec_eval-8.0/Makefile diff --git a/sm-model/trec_eval-8.0/README b/sm_model/trec_eval-8.0/README similarity index 100% rename from sm-model/trec_eval-8.0/README rename to sm_model/trec_eval-8.0/README diff --git a/sm-model/trec_eval-8.0/buf.h b/sm_model/trec_eval-8.0/buf.h similarity index 100% rename from sm-model/trec_eval-8.0/buf.h rename to sm_model/trec_eval-8.0/buf.h diff --git a/sm-model/trec_eval-8.0/buf_util.c b/sm_model/trec_eval-8.0/buf_util.c similarity index 100% rename from sm-model/trec_eval-8.0/buf_util.c rename to sm_model/trec_eval-8.0/buf_util.c diff --git a/sm-model/trec_eval-8.0/common.h b/sm_model/trec_eval-8.0/common.h similarity index 100% rename from sm-model/trec_eval-8.0/common.h rename to sm_model/trec_eval-8.0/common.h diff --git a/sm-model/trec_eval-8.0/error_msgs.c b/sm_model/trec_eval-8.0/error_msgs.c similarity index 100% rename from sm-model/trec_eval-8.0/error_msgs.c rename to sm_model/trec_eval-8.0/error_msgs.c diff --git a/sm-model/trec_eval-8.0/form_trvec.c b/sm_model/trec_eval-8.0/form_trvec.c similarity index 100% rename from sm-model/trec_eval-8.0/form_trvec.c rename to sm_model/trec_eval-8.0/form_trvec.c diff --git a/sm-model/trec_eval-8.0/get_qrels.c b/sm_model/trec_eval-8.0/get_qrels.c similarity index 100% rename from sm-model/trec_eval-8.0/get_qrels.c rename to sm_model/trec_eval-8.0/get_qrels.c diff --git a/sm-model/trec_eval-8.0/get_top.c b/sm_model/trec_eval-8.0/get_top.c similarity index 100% rename from sm-model/trec_eval-8.0/get_top.c rename to sm_model/trec_eval-8.0/get_top.c diff --git a/sm-model/trec_eval-8.0/smart_error.h b/sm_model/trec_eval-8.0/smart_error.h similarity index 100% rename from sm-model/trec_eval-8.0/smart_error.h rename to sm_model/trec_eval-8.0/smart_error.h diff --git a/sm-model/trec_eval-8.0/sysfunc.h b/sm_model/trec_eval-8.0/sysfunc.h similarity index 100% rename from sm-model/trec_eval-8.0/sysfunc.h rename to sm_model/trec_eval-8.0/sysfunc.h diff --git a/sm-model/trec_eval-8.0/tr_vec.h b/sm_model/trec_eval-8.0/tr_vec.h similarity index 100% rename from sm-model/trec_eval-8.0/tr_vec.h rename to sm_model/trec_eval-8.0/tr_vec.h diff --git a/sm-model/trec_eval-8.0/trec_eval.c b/sm_model/trec_eval-8.0/trec_eval.c similarity index 100% rename from sm-model/trec_eval-8.0/trec_eval.c rename to sm_model/trec_eval-8.0/trec_eval.c diff --git a/sm-model/utils.py b/sm_model/utils.py similarity index 82% rename from sm-model/utils.py rename to sm_model/utils.py index da9fd46..064828e 100644 --- a/sm-model/utils.py +++ b/sm_model/utils.py @@ -1,4 +1,4 @@ -# file input output +# file input output import os import sys @@ -27,7 +27,8 @@ def logargs(func): def cache_word_embeddings(word_embeddings_file, cache_file): if not word_embeddings_file.endswith('.gz'): - logger.warning('WARNING: expecting a .gz file. Is the {} in the correct format?'.format(word_embeddings_file)) + logger.warning('WARNING: expecting a .gz file. Is the {} in the correct \ + format?'.format(word_embeddings_file)) vocab_size, vec_dim = 0, 0 @@ -36,8 +37,7 @@ def cache_word_embeddings(word_embeddings_file, cache_file): if not os.path.exists(os.path.dirname(cache_file)): # make cache folder if needed os.mkdir(os.path.dirname(cache_file)) - - logger.info('caching the word embeddings in np.memmap format') + logger.info('caching the word embeddings in np.memmap format') wv = KeyedVectors.load_word2vec_format(word_embeddings_file, binary=True) # print len(wv.syn0), wv.syn0.shape # print len(wv.syn0norm) if wv.syn0norm else None @@ -50,7 +50,7 @@ def cache_word_embeddings(word_embeddings_file, cache_file): with open(cache_file + '.dimensions', 'w') as f: logger.info('writing out dimensions for {}'.format(word_embeddings_file)) print(wv.syn0.shape[0], wv.syn0.shape[1], file=f) - vocab_size, vec_dim = wv.syn0.shape + vocab_size, vec_dim = wv.syn0.shape del fp, wv print('cached {} into {}'.format(word_embeddings_file, cache_file)) @@ -63,9 +63,8 @@ def load_embedding_dimensions(cache_file): vocab_size, vec_dim = [int(e) for e in d.read().strip().split()] return vocab_size, vec_dim - -def load_cached_embeddings(cache_file, vocab_list, oov_vec = []): - logger.debug('loading cached embeddings ') +def load_cached_embeddings(cache_file, vocab_list, oov_vec=[]): + logger.debug('loading cached embeddings ') with open(cache_file + '.dimensions') as d: vocab_size, vec_dim = [int(e) for e in d.read().strip().split()] @@ -76,8 +75,8 @@ def load_cached_embeddings(cache_file, vocab_list, oov_vec = []): logger.debug('loading vocab') w2v_vocab_list = map(str.strip, f.readlines()) - vocab_dict = {w:k for k,w in enumerate(w2v_vocab_list)} - + vocab_dict = {w:k for k, w in enumerate(w2v_vocab_list)} + # Read w2v for vocab appears in Q and A w2v_dict = {} for word in vocab_list: @@ -86,7 +85,8 @@ def load_cached_embeddings(cache_file, vocab_list, oov_vec = []): if word in vocab_dict: w2v_dict[word] = W[vocab_dict[word]] else: - w2v_dict[word] = np.random.uniform(-0.25, 0.25, vec_dim) if len(oov_vec) == 0 else oov_vec + w2v_dict[word] = np.random.uniform(-0.25, 0.25, vec_dim) \ + if len(oov_vec) == 0 else oov_vec #w2v_dict[word] = W[vocab_dict["unk"]] return w2v_dict @@ -99,12 +99,16 @@ def read_in_dataset(dataset_folder, set_folder): max_q = 0 max_s = 0 set_path = os.path.join(dataset_folder, set_folder) - len_q_list =[len(line.strip().split()) for line in open(os.path.join(set_path, 'a.toks')).readlines()] + questions = [line.strip() for line in open(os.path.join(set_path, 'a.toks')).readlines()] - len_s_list =[len(line.strip().split()) for line in open(os.path.join(set_path, 'b.toks')).readlines()] + len_q_list = [len(q.split()) for q in questions] + sentences = [line.strip() for line in open(os.path.join(set_path, 'b.toks')).readlines()] + len_s_list = [len(s.split()) for s in sentences] + labels = [int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines()] - ext_feats = np.array([list(map(float, line.strip().split(' '))) for line in open(os.path.join(set_path, 'overlap_feats.txt')).readlines()]) + ext_feats = np.array([list(map(float, line.strip().split(' '))) \ + for line in open(os.path.join(set_path, 'overlap_feats.txt')).readlines()]) #y = torch.from_numpy(labels) #return questions, sentences, y @@ -121,11 +125,11 @@ def get_test_qids_labels(dataset_folder, set_folder): if __name__ == "__main__": - + vocab = ["unk", "idontreallythinkthiswordexists", "hello"] - w2v_dict, vec_dim = load_cached_embeddings("../../data/word2vec-models/aquaint+wiki.txt.gz.ndim=50.cache", vocab) - + w2v_dict, vec_dim = load_cached_embeddings("../../data/word2vec/aquaint+wiki.txt.gz.ndim=50.cache", vocab) + for w, v in w2v_dict.iteritems(): print(w) print(v)